mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-24 16:42:05 +00:00
Merge remote-tracking branch 'origin/master' into HEAD
This commit is contained in:
Alexander Kuzmenkov
commit
4dc6254357
@ -131,5 +131,5 @@ done
|
||||
|
||||
dmesg -T > dmesg.log
|
||||
|
||||
7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze
|
||||
7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze benchmark
|
||||
cp compare.log /output
|
||||
|
||||
@ -316,7 +316,7 @@ Result:
|
||||
The function takes as arguments a set of conditions from 1 to 32 arguments of type `UInt8` that indicate whether a certain condition was met for the event.
|
||||
Any condition can be specified as an argument (as in [WHERE](../../sql-reference/statements/select/where.md#select-where)).
|
||||
|
||||
The conditions, except the first, apply in pairs: the result of the second will be true if the first and second are true, of the third if the first and fird are true, etc.
|
||||
The conditions, except the first, apply in pairs: the result of the second will be true if the first and second are true, of the third if the first and third are true, etc.
|
||||
|
||||
**Syntax**
|
||||
|
||||
|
||||
@ -1805,7 +1805,7 @@ For more information see [parameters](#agg_functions-stochasticlinearregression-
|
||||
stochasticLogisticRegression(1.0, 1.0, 10, 'SGD')
|
||||
```
|
||||
|
||||
1. Fitting
|
||||
**1.** Fitting
|
||||
|
||||
<!-- -->
|
||||
|
||||
@ -1813,7 +1813,7 @@ stochasticLogisticRegression(1.0, 1.0, 10, 'SGD')
|
||||
|
||||
Predicted labels have to be in \[-1, 1\].
|
||||
|
||||
1. Predicting
|
||||
**2.** Predicting
|
||||
|
||||
<!-- -->
|
||||
|
||||
|
||||
@ -28,7 +28,7 @@ CREATE TABLE t
|
||||
) ENGINE = ...
|
||||
```
|
||||
|
||||
[uniq](../../sql-reference/aggregate-functions/reference.md#agg_function-uniq), anyIf ([any](../../sql-reference/aggregate-functions/reference.md#agg_function-any)+[If](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-if)) and [quantiles](../../sql-reference/aggregate-functions/reference.md) are the aggregate functions supported in ClickHouse.
|
||||
[uniq](../../sql-reference/aggregate-functions/reference.md#agg_function-uniq), anyIf ([any](../../sql-reference/aggregate-functions/reference.md#agg_function-any)+[If](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-if)) and [quantiles](../../sql-reference/aggregate-functions/reference.md#quantiles) are the aggregate functions supported in ClickHouse.
|
||||
|
||||
## Usage {#usage}
|
||||
|
||||
|
||||
@ -220,7 +220,7 @@ if __name__ == '__main__':
|
||||
arg_parser.add_argument('--website-dir', default=website_dir)
|
||||
arg_parser.add_argument('--output-dir', default='build')
|
||||
arg_parser.add_argument('--enable-stable-releases', action='store_true')
|
||||
arg_parser.add_argument('--stable-releases-limit', type=int, default='4')
|
||||
arg_parser.add_argument('--stable-releases-limit', type=int, default='3')
|
||||
arg_parser.add_argument('--lts-releases-limit', type=int, default='2')
|
||||
arg_parser.add_argument('--nav-limit', type=int, default='0')
|
||||
arg_parser.add_argument('--version-prefix', type=str, default='')
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
Babel==2.8.0
|
||||
certifi==2020.4.5.2
|
||||
chardet==3.0.4
|
||||
googletrans==2.4.0
|
||||
googletrans==3.0.0
|
||||
idna==2.9
|
||||
Jinja2==2.11.2
|
||||
pandocfilters==1.4.2
|
||||
|
||||
@ -1,15 +1,13 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_priority: 31
|
||||
toc_title: "\u61D2\u60F0"
|
||||
---
|
||||
|
||||
# 懒惰 {#lazy}
|
||||
# 延时引擎Lazy {#lazy}
|
||||
|
||||
仅将表保留在RAM中 `expiration_time_in_seconds` 上次访问后几秒钟。 只能与\*日志表一起使用。
|
||||
在距最近一次访问间隔`expiration_time_in_seconds`时间段内，将表保存在内存中，仅适用于 \*Log引擎表
|
||||
|
||||
它针对存储许多小\*日志表进行了优化，访问之间存在较长的时间间隔。
|
||||
由于针对这类表的访问间隔较长，对保存大量小的 \*Log引擎表进行了优化，
|
||||
|
||||
## 创建数据库 {#creating-a-database}
|
||||
|
||||
|
||||
@ -1,11 +1,9 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_priority: 61
|
||||
toc_title: "\uFF82\u6697\uFF6A\uFF82\u6C3E\u73AF\u50AC\uFF82\u56E3"
|
||||
toc_title: "性能测试"
|
||||
---
|
||||
|
||||
# ﾂ暗ｪﾂ氾环催ﾂ団 {#clickhouse-benchmark}
|
||||
# 性能测试 {#clickhouse-benchmark}
|
||||
|
||||
连接到ClickHouse服务器并重复发送指定的查询。
|
||||
|
||||
@ -21,7 +19,7 @@ $ echo "single query" | clickhouse-benchmark [keys]
|
||||
$ clickhouse-benchmark [keys] <<< "single query"
|
||||
```
|
||||
|
||||
如果要发送一组查询，请创建一个文本文件，并将每个查询放在此文件中的单个字符串上。 例如:
|
||||
如果要发送一组查询，请创建一个文本文件，并将每个查询的字符串放在此文件中。 例如:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM system.numbers LIMIT 10000000
|
||||
@ -34,15 +32,15 @@ SELECT 1
|
||||
clickhouse-benchmark [keys] < queries_file
|
||||
```
|
||||
|
||||
## 键 {#clickhouse-benchmark-keys}
|
||||
## keys参数 {#clickhouse-benchmark-keys}
|
||||
|
||||
- `-c N`, `--concurrency=N` — Number of queries that `clickhouse-benchmark` 同时发送。 默认值：1。
|
||||
- `-d N`, `--delay=N` — Interval in seconds between intermediate reports (set 0 to disable reports). Default value: 1.
|
||||
- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. 为 [比较模式](#clickhouse-benchmark-comparison-mode) 您可以使用多个 `-h` 钥匙
|
||||
- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. 为 [比较模式](#clickhouse-benchmark-comparison-mode) 您可以使用多个 `-h` 参数
|
||||
- `-p N`, `--port=N` — Server port. Default value: 9000. For the [比较模式](#clickhouse-benchmark-comparison-mode) 您可以使用多个 `-p` 钥匙
|
||||
- `-i N`, `--iterations=N` — Total number of queries. Default value: 0.
|
||||
- `-r`, `--randomize` — Random order of queries execution if there is more then one input query.
|
||||
- `-s`, `--secure` — Using TLS connection.
|
||||
- `-i N`, `--iterations=N` — 查询的总次数. Default value: 0.
|
||||
- `-r`, `--randomize` — 有多个查询时，以随机顺序执行.
|
||||
- `-s`, `--secure` — 使用TLS安全连接.
|
||||
- `-t N`, `--timelimit=N` — Time limit in seconds. `clickhouse-benchmark` 达到指定的时间限制时停止发送查询。 默认值：0（禁用时间限制）。
|
||||
- `--confidence=N` — Level of confidence for T-test. Possible values: 0 (80%), 1 (90%), 2 (95%), 3 (98%), 4 (99%), 5 (99.5%). Default value: 5. In the [比较模式](#clickhouse-benchmark-comparison-mode) `clickhouse-benchmark` 执行 [独立双样本学生的t测试](https://en.wikipedia.org/wiki/Student%27s_t-test#Independent_two-sample_t-test) 测试以确定两个分布是否与所选置信水平没有不同。
|
||||
- `--cumulative` — Printing cumulative data instead of data per interval.
|
||||
@ -51,14 +49,14 @@ clickhouse-benchmark [keys] < queries_file
|
||||
- `--user=USERNAME` — ClickHouse user name. Default value: `default`.
|
||||
- `--password=PSWD` — ClickHouse user password. Default value: empty string.
|
||||
- `--stacktrace` — Stack traces output. When the key is set, `clickhouse-bencmark` 输出异常的堆栈跟踪。
|
||||
- `--stage=WORD` — Query processing stage at server. ClickHouse stops query processing and returns answer to `clickhouse-benchmark` 在指定的阶段。 可能的值: `complete`, `fetch_columns`, `with_mergeable_state`. 默认值: `complete`.
|
||||
- `--stage=WORD` — 查询请求的服务端处理状态. 在特定阶段Clickhouse会停止查询处理，并返回结果给`clickhouse-benchmark`。 可能的值: `complete`, `fetch_columns`, `with_mergeable_state`. 默认值: `complete`.
|
||||
- `--help` — Shows the help message.
|
||||
|
||||
如果你想申请一些 [设置](../../operations/settings/index.md) 对于查询，请将它们作为键传递 `--<session setting name>= SETTING_VALUE`. 例如, `--max_memory_usage=1048576`.
|
||||
如果你想在查询时应用上述的部分参数 [设置](../../operations/settings/index.md) ，请将它们作为键传递 `--<session setting name>= SETTING_VALUE`. 例如, `--max_memory_usage=1048576`.
|
||||
|
||||
## 输出 {#clickhouse-benchmark-output}
|
||||
|
||||
默认情况下, `clickhouse-benchmark` 每个报表 `--delay` 间隔。
|
||||
默认情况下, `clickhouse-benchmark` 按照 `--delay` 参数间隔输出结果。
|
||||
|
||||
报告示例:
|
||||
|
||||
@ -83,27 +81,27 @@ localhost:9000, queries 10, QPS: 6.772, RPS: 67904487.440, MiB/s: 518.070, resul
|
||||
99.990% 0.150 sec.
|
||||
```
|
||||
|
||||
在报告中，您可以找到:
|
||||
在结果报告中，您可以找到:
|
||||
|
||||
- 在查询的数量 `Queries executed:` 场。
|
||||
- 查询数量：参见`Queries executed:`字段。
|
||||
|
||||
- 状态字符串包含（按顺序):
|
||||
- 状态码（按顺序给出）:
|
||||
|
||||
- ClickHouse服务器的端点。
|
||||
- ClickHouse服务器的连接信息。
|
||||
- 已处理的查询数。
|
||||
- QPS：QPS：在指定的时间段内每秒执行多少个查询服务器 `--delay` 争论。
|
||||
- RPS：在指定的时间段内，服务器每秒读取多少行 `--delay` 争论。
|
||||
- MiB/s：在指定的时间段内每秒读取多少mebibytes服务器 `--delay` 争论。
|
||||
- 结果RPS：在指定的时间段内，服务器每秒放置到查询结果的行数 `--delay` 争论。
|
||||
- 结果MiB/s.在指定的时间段内，服务器每秒将多少mebibytes放置到查询结果中 `--delay` 争论。
|
||||
- QPS：服务端每秒处理的查询数量
|
||||
- RPS：服务器每秒读取多少行
|
||||
- MiB/s：服务器每秒读取多少字节的数据
|
||||
- 结果RPS：服务端每秒生成多少行的结果集数据
|
||||
- 结果MiB/s.服务端每秒生成多少字节的结果集数据
|
||||
|
||||
- 查询执行时间的百分位数。
|
||||
- 查询执行时间的百分比。
|
||||
|
||||
## 比较模式 {#clickhouse-benchmark-comparison-mode}
|
||||
## 对比模式 {#clickhouse-benchmark-comparison-mode}
|
||||
|
||||
`clickhouse-benchmark` 可以比较两个正在运行的ClickHouse服务器的性能。
|
||||
|
||||
要使用比较模式，请通过以下两对指定两个服务器的端点 `--host`, `--port` 钥匙 键在参数列表中的位置匹配在一起，第一 `--host` 与第一匹配 `--port` 等等。 `clickhouse-benchmark` 建立到两个服务器的连接，然后发送查询。 每个查询寻址到随机选择的服务器。 每个服务器的结果分别显示。
|
||||
要使用对比模式，分别为每个服务器配置各自的`--host`, `--port`参数。`clickhouse-benchmark` 会根据设置的参数建立到各个Server的连接并发送请求。每个查询请求会随机发送到某个服务器。输出结果会按服务器分组输出
|
||||
|
||||
## 示例 {#clickhouse-benchmark-example}
|
||||
|
||||
|
||||
@ -1,51 +1,49 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_priority: 37
|
||||
toc_title: "\u7EC4\u5408\u5668"
|
||||
toc_title: 聚合函数组合器
|
||||
---
|
||||
|
||||
# 聚合函数组合器 {#aggregate_functions_combinators}
|
||||
|
||||
聚合函数的名称可以附加一个后缀。 这改变了聚合函数的工作方式。
|
||||
|
||||
## -如果 {#agg-functions-combinator-if}
|
||||
## -If {#agg-functions-combinator-if}
|
||||
|
||||
The suffix -If can be appended to the name of any aggregate function. In this case, the aggregate function accepts an extra argument – a condition (Uint8 type). The aggregate function processes only the rows that trigger the condition. If the condition was not triggered even once, it returns a default value (usually zeros or empty strings).
|
||||
-If可以加到任何聚合函数之后。加了-If之后聚合函数需要接受一个额外的参数，一个条件（Uint8类型），如果条件满足，那聚合函数处理当前的行数据，如果不满足，那返回默认值（通常是0或者空字符串）。
|
||||
|
||||
例: `sumIf(column, cond)`, `countIf(cond)`, `avgIf(x, cond)`, `quantilesTimingIf(level1, level2)(x, cond)`, `argMinIf(arg, val, cond)` 等等。
|
||||
例： `sumIf(column, cond)`, `countIf(cond)`, `avgIf(x, cond)`, `quantilesTimingIf(level1, level2)(x, cond)`, `argMinIf(arg, val, cond)` 等等。
|
||||
|
||||
使用条件聚合函数，您可以一次计算多个条件的聚合，而无需使用子查询和 `JOIN`例如，在Yandex的。Metrica，条件聚合函数用于实现段比较功能。
|
||||
使用条件聚合函数，您可以一次计算多个条件的聚合，而无需使用子查询和 `JOIN`例如，在Yandex.Metrica，条件聚合函数用于实现段比较功能。
|
||||
|
||||
## -阵列 {#agg-functions-combinator-array}
|
||||
## -Array {#agg-functions-combinator-array}
|
||||
|
||||
-Array后缀可以附加到任何聚合函数。 在这种情况下，聚合函数采用的参数 ‘Array(T)’ 类型（数组）而不是 ‘T’ 类型参数。 如果聚合函数接受多个参数，则它必须是长度相等的数组。 在处理数组时，聚合函数的工作方式与所有数组元素的原始聚合函数类似。
|
||||
|
||||
示例1: `sumArray(arr)` -总计所有的所有元素 ‘arr’ 阵列。 在这个例子中，它可以更简单地编写: `sum(arraySum(arr))`.
|
||||
示例1： `sumArray(arr)` -总计所有的所有元素 ‘arr’ 阵列。在这个例子中，它可以更简单地编写: `sum(arraySum(arr))`.
|
||||
|
||||
示例2: `uniqArray(arr)` – Counts the number of unique elements in all ‘arr’ 阵列。 这可以做一个更简单的方法: `uniq(arrayJoin(arr))`，但它并不总是可以添加 ‘arrayJoin’ 到查询。
|
||||
示例2： `uniqArray(arr)` – 计算‘arr’中唯一元素的个数。这可以是一个更简单的方法： `uniq(arrayJoin(arr))`，但它并不总是可以添加 ‘arrayJoin’ 到查询。
|
||||
|
||||
-如果和-阵列可以组合。 然而, ‘Array’ 必须先来，然后 ‘If’. 例: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. 由于这个顺序，该 ‘cond’ 参数不会是数组。
|
||||
如果和-If组合，‘Array’ 必须先来，然后 ‘If’. 例： `uniqArrayIf(arr, cond)`， `quantilesTimingArrayIf(level1, level2)(arr, cond)`。由于这个顺序，该 ‘cond’ 参数不会是数组。
|
||||
|
||||
## -州 {#agg-functions-combinator-state}
|
||||
## -State {#agg-functions-combinator-state}
|
||||
|
||||
如果应用此combinator，则聚合函数不会返回结果值（例如唯一值的数量 [uniq](reference.md#agg_function-uniq) 函数），但聚合的中间状态（用于 `uniq`，这是用于计算唯一值的数量的散列表）。 这是一个 `AggregateFunction(...)` 可用于进一步处理或存储在表中以完成聚合。
|
||||
如果应用此combinator，则聚合函数不会返回结果值（例如唯一值的数量 [uniq](reference.md#agg_function-uniq) 函数），但是返回聚合的中间状态（对于 `uniq`，返回的是计算唯一值的数量的哈希表）。 这是一个 `AggregateFunction(...)` 可用于进一步处理或存储在表中以完成稍后的聚合。
|
||||
|
||||
要使用这些状态，请使用:
|
||||
|
||||
- [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) 表引擎。
|
||||
- [最后聚会](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) 功能。
|
||||
- [跑累积](../../sql-reference/functions/other-functions.md#function-runningaccumulate) 功能。
|
||||
- [-合并](#aggregate_functions_combinators-merge) combinator
|
||||
- [finalizeAggregation](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) 功能。
|
||||
- [runningAccumulate](../../sql-reference/functions/other-functions.md#function-runningaccumulate) 功能。
|
||||
- [-Merge](#aggregate_functions_combinators-merge) combinator
|
||||
- [-MergeState](#aggregate_functions_combinators-mergestate) combinator
|
||||
|
||||
## -合并 {#aggregate_functions_combinators-merge}
|
||||
## -Merge {#aggregate_functions_combinators-merge}
|
||||
|
||||
如果应用此组合器，则聚合函数将中间聚合状态作为参数，组合状态以完成聚合，并返回结果值。
|
||||
|
||||
## -MergeState {#aggregate_functions_combinators-mergestate}
|
||||
|
||||
以与-Merge combinator相同的方式合并中间聚合状态。 但是，它不会返回结果值，而是返回中间聚合状态，类似于-State combinator。
|
||||
以与-Merge 相同的方式合并中间聚合状态。 但是，它不会返回结果值，而是返回中间聚合状态，类似于-State。
|
||||
|
||||
## -ForEach {#agg-functions-combinator-foreach}
|
||||
|
||||
@ -55,7 +53,7 @@ The suffix -If can be appended to the name of any aggregate function. In this ca
|
||||
|
||||
更改聚合函数的行为。
|
||||
|
||||
如果聚合函数没有输入值，则使用此combinator，它返回其返回数据类型的默认值。 适用于可以采用空输入数据的聚合函数。
|
||||
如果聚合函数没有输入值，则使用此组合器它返回其返回数据类型的默认值。 适用于可以采用空输入数据的聚合函数。
|
||||
|
||||
`-OrDefault` 可与其他组合器一起使用。
|
||||
|
||||
@ -67,7 +65,7 @@ The suffix -If can be appended to the name of any aggregate function. In this ca
|
||||
|
||||
**参数**
|
||||
|
||||
- `x` — Aggregate function parameters.
|
||||
- `x` — 聚合函数参数。
|
||||
|
||||
**返回值**
|
||||
|
||||
@ -174,7 +172,7 @@ FROM
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
## -重新采样 {#agg-functions-combinator-resample}
|
||||
## -Resample {#agg-functions-combinator-resample}
|
||||
|
||||
允许您将数据划分为组，然后单独聚合这些组中的数据。 通过将一列中的值拆分为间隔来创建组。
|
||||
|
||||
@ -184,19 +182,19 @@ FROM
|
||||
|
||||
**参数**
|
||||
|
||||
- `start` — Starting value of the whole required interval for `resampling_key` 值。
|
||||
- `stop` — Ending value of the whole required interval for `resampling_key` 值。 整个时间间隔不包括 `stop` 价值 `[start, stop)`.
|
||||
- `step` — Step for separating the whole interval into subintervals. The `aggFunction` 在每个子区间上独立执行。
|
||||
- `resampling_key` — Column whose values are used for separating data into intervals.
|
||||
- `start` — `resampling_key` 开始值。
|
||||
- `stop` — `resampling_key` 结束边界。 区间内部不包含 `stop` 值，即 `[start, stop)`.
|
||||
- `step` — 分组的步长。 The `aggFunction` 在每个子区间上独立执行。
|
||||
- `resampling_key` — 取样列，被用来分组.
|
||||
- `aggFunction_params` — `aggFunction` 参数。
|
||||
|
||||
**返回值**
|
||||
|
||||
- 阵列 `aggFunction` 每个子区间的结果。
|
||||
- `aggFunction` 每个子区间的结果，结果为数组。
|
||||
|
||||
**示例**
|
||||
|
||||
考虑一下 `people` 具有以下数据的表:
|
||||
考虑一下 `people` 表具有以下数据的表结构：
|
||||
|
||||
``` text
|
||||
┌─name───┬─age─┬─wage─┐
|
||||
@ -209,9 +207,9 @@ FROM
|
||||
└────────┴─────┴──────┘
|
||||
```
|
||||
|
||||
让我们得到的人的名字，他们的年龄在于的时间间隔 `[30,60)` 和 `[60,75)`. 由于我们使用整数表示的年龄，我们得到的年龄 `[30, 59]` 和 `[60,74]` 间隔。
|
||||
让我们得到的人的名字，他们的年龄在于的时间间隔 `[30,60)` 和 `[60,75)`。 由于我们使用整数表示的年龄，我们得到的年龄 `[30, 59]` 和 `[60,74]` 间隔。
|
||||
|
||||
要在数组中聚合名称，我们使用 [groupArray](reference.md#agg_function-grouparray) 聚合函数。 这需要一个参数。 在我们的例子中，它是 `name` 列。 该 `groupArrayResample` 函数应该使用 `age` 按年龄聚合名称的列。 要定义所需的时间间隔，我们通过 `30, 75, 30` 参数到 `groupArrayResample` 功能。
|
||||
要在数组中聚合名称，我们使用 [groupArray](reference.md#agg_function-grouparray) 聚合函数。 这需要一个参数。 在我们的例子中，它是 `name` 列。 `groupArrayResample` 函数应该使用 `age` 按年龄聚合名称， 要定义所需的时间间隔，我们传入 `30, 75, 30` 参数给 `groupArrayResample` 函数。
|
||||
|
||||
``` sql
|
||||
SELECT groupArrayResample(30, 75, 30)(name, age) FROM people
|
||||
@ -225,7 +223,7 @@ SELECT groupArrayResample(30, 75, 30)(name, age) FROM people
|
||||
|
||||
考虑结果。
|
||||
|
||||
`Jonh` 是因为他太年轻了 其他人按照指定的年龄间隔进行分配。
|
||||
`Jonh` 没有被选中，因为他太年轻了。 其他人按照指定的年龄间隔进行分配。
|
||||
|
||||
现在让我们计算指定年龄间隔内的总人数和平均工资。
|
||||
|
||||
|
||||
@ -1,9 +1,6 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_folder_title: "\u805A\u5408\u51FD\u6570"
|
||||
toc_priority: 33
|
||||
toc_title: "\u5BFC\u8A00"
|
||||
toc_title: 简介
|
||||
---
|
||||
|
||||
# 聚合函数 {#aggregate-functions}
|
||||
|
||||
@ -1,15 +1,13 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_priority: 38
|
||||
toc_title: "\u53C2\u6570"
|
||||
toc_title: 参数聚合函数
|
||||
---
|
||||
|
||||
# 参数聚合函数 {#aggregate_functions_parametric}
|
||||
|
||||
Some aggregate functions can accept not only argument columns (used for compression), but a set of parameters – constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments.
|
||||
一些聚合函数不仅可以接受参数列（用于压缩），也可以接收常量的初始化参数。这种语法是接受两个括号的参数，第一个数初始化参数，第二个是入参。
|
||||
|
||||
## 直方图 {#histogram}
|
||||
## histogram {#histogram}
|
||||
|
||||
计算自适应直方图。 它不能保证精确的结果。
|
||||
|
||||
@ -21,20 +19,21 @@ histogram(number_of_bins)(values)
|
||||
|
||||
**参数**
|
||||
|
||||
`number_of_bins` — Upper limit for the number of bins in the histogram. The function automatically calculates the number of bins. It tries to reach the specified number of bins, but if it fails, it uses fewer bins.
|
||||
`values` — [表达式](../syntax.md#syntax-expressions) 导致输入值。
|
||||
`number_of_bins` — 直方图bin个数，这个函数会自动计算bin的数量，而且会尽量使用指定值，如果无法做到，那就使用更小的bin个数。
|
||||
|
||||
`values` — [表达式](../syntax.md#syntax-expressions) 输入值。
|
||||
|
||||
**返回值**
|
||||
|
||||
- [阵列](../../sql-reference/data-types/array.md) 的 [元组](../../sql-reference/data-types/tuple.md) 下面的格式:
|
||||
- [Array](../../sql-reference/data-types/array.md) 的 [Tuples](../../sql-reference/data-types/tuple.md) 如下：
|
||||
|
||||
```
|
||||
[(lower_1, upper_1, height_1), ... (lower_N, upper_N, height_N)]
|
||||
```
|
||||
|
||||
- `lower` — Lower bound of the bin.
|
||||
- `upper` — Upper bound of the bin.
|
||||
- `height` — Calculated height of the bin.
|
||||
- `lower` — bin的下边界。
|
||||
- `upper` — bin的上边界。
|
||||
- `height` — bin的计算权重。
|
||||
|
||||
**示例**
|
||||
|
||||
@ -53,7 +52,7 @@ FROM (
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
您可以使用 [酒吧](../../sql-reference/functions/other-functions.md#function-bar) 功能，例如:
|
||||
您可以使用 [bar](../../sql-reference/functions/other-functions.md#function-bar) 功能，例如:
|
||||
|
||||
``` sql
|
||||
WITH histogram(5)(rand() % 100) AS hist
|
||||
@ -93,11 +92,11 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...)
|
||||
|
||||
**参数**
|
||||
|
||||
- `pattern` — Pattern string. See [模式语法](#sequence-function-pattern-syntax).
|
||||
- `pattern` — 模式字符串。 参考 [模式语法](#sequence-function-pattern-syntax).
|
||||
|
||||
- `timestamp` — Column considered to contain time data. Typical data types are `Date` 和 `DateTime`. 您还可以使用任何支持的 [UInt](../../sql-reference/data-types/int-uint.md) 数据类型。
|
||||
- `timestamp` — 包含时间的列。典型的时间类型是： `Date` 和 `DateTime`。您还可以使用任何支持的 [UInt](../../sql-reference/data-types/int-uint.md) 数据类型。
|
||||
|
||||
- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. 最多可以传递32个条件参数。 该函数只考虑这些条件中描述的事件。 如果序列包含未在条件中描述的数据，则函数将跳过这些数据。
|
||||
- `cond1`, `cond2` — 事件链的约束条件。 数据类型是： `UInt8`。 最多可以传递32个条件参数。 该函数只考虑这些条件中描述的事件。 如果序列包含未在条件中描述的数据，则函数将跳过这些数据。
|
||||
|
||||
**返回值**
|
||||
|
||||
@ -109,11 +108,11 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...)
|
||||
<a name="sequence-function-pattern-syntax"></a>
|
||||
**模式语法**
|
||||
|
||||
- `(?N)` — Matches the condition argument at position `N`. 条件在编号 `[1, 32]` 范围。 例如, `(?1)` 匹配传递给 `cond1` 参数。
|
||||
- `(?N)` — 在位置`N`匹配条件参数。 条件在编号 `[1, 32]` 范围。 例如, `(?1)` 匹配传递给 `cond1` 参数。
|
||||
|
||||
- `.*` — Matches any number of events. You don't need conditional arguments to match this element of the pattern.
|
||||
- `.*` — 匹配任何事件的数字。 不需要条件参数来匹配这个模式。
|
||||
|
||||
- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` 匹配彼此发生超过1800秒的事件。 这些事件之间可以存在任意数量的任何事件。 您可以使用 `>=`, `>`, `<`, `<=` 运营商。
|
||||
- `(?t operator value)` — 分开两个事件的时间。 例如： `(?1)(?t>1800)(?2)` 匹配彼此发生超过1800秒的事件。 这些事件之间可以存在任意数量的任何事件。 您可以使用 `>=`, `>`, `<`, `<=` 运算符。
|
||||
|
||||
**例**
|
||||
|
||||
@ -169,7 +168,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM
|
||||
|
||||
## sequenceCount(pattern)(time, cond1, cond2, …) {#function-sequencecount}
|
||||
|
||||
计数与模式匹配的事件链的数量。 该函数搜索不重叠的事件链。 当前链匹配后，它开始搜索下一个链。
|
||||
计算与模式匹配的事件链的数量。该函数搜索不重叠的事件链。当前链匹配后，它开始搜索下一个链。
|
||||
|
||||
!!! warning "警告"
|
||||
在同一秒钟发生的事件可能以未定义的顺序排列在序列中，影响结果。
|
||||
@ -180,11 +179,11 @@ sequenceCount(pattern)(timestamp, cond1, cond2, ...)
|
||||
|
||||
**参数**
|
||||
|
||||
- `pattern` — Pattern string. See [模式语法](#sequence-function-pattern-syntax).
|
||||
- `pattern` — 模式字符串。 参考：[模式语法](#sequence-function-pattern-syntax).
|
||||
|
||||
- `timestamp` — Column considered to contain time data. Typical data types are `Date` 和 `DateTime`. 您还可以使用任何支持的 [UInt](../../sql-reference/data-types/int-uint.md) 数据类型。
|
||||
- `timestamp` — 包含时间的列。典型的时间类型是： `Date` 和 `DateTime`。您还可以使用任何支持的 [UInt](../../sql-reference/data-types/int-uint.md) 数据类型。
|
||||
|
||||
- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. 最多可以传递32个条件参数。 该函数只考虑这些条件中描述的事件。 如果序列包含未在条件中描述的数据，则函数将跳过这些数据。
|
||||
- `cond1`, `cond2` — 事件链的约束条件。 数据类型是： `UInt8`。 最多可以传递32个条件参数。该函数只考虑这些条件中描述的事件。 如果序列包含未在条件中描述的数据，则函数将跳过这些数据。
|
||||
|
||||
**返回值**
|
||||
|
||||
@ -227,9 +226,9 @@ SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t
|
||||
|
||||
搜索滑动时间窗中的事件链，并计算从链中发生的最大事件数。
|
||||
|
||||
该函数根据算法工作:
|
||||
该函数采用如下算法：
|
||||
|
||||
- 该函数搜索触发链中的第一个条件并将事件计数器设置为1的数据。 这是滑动窗口启动的时刻。
|
||||
- 该函数搜索触发链中的第一个条件并将事件计数器设置为1。 这是滑动窗口启动的时刻。
|
||||
|
||||
- 如果来自链的事件在窗口内顺序发生，则计数器将递增。 如果事件序列中断，则计数器不会增加。
|
||||
|
||||
@ -243,11 +242,11 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN)
|
||||
|
||||
**参数**
|
||||
|
||||
- `window` — Length of the sliding window in seconds.
|
||||
- `mode` -这是一个可选的参数。
|
||||
- `'strict'` -当 `'strict'` 设置时，windowFunnel()仅对唯一值应用条件。
|
||||
- `timestamp` — Name of the column containing the timestamp. Data types supported: [日期](../../sql-reference/data-types/date.md), [日期时间](../../sql-reference/data-types/datetime.md#data_type-datetime) 和其他无符号整数类型（请注意，即使时间戳支持 `UInt64` 类型，它的值不能超过Int64最大值，即2^63-1）。
|
||||
- `cond` — Conditions or data describing the chain of events. [UInt8](../../sql-reference/data-types/int-uint.md).
|
||||
- `window` — 滑动窗户的大小，单位是秒。
|
||||
- `mode` - 这是一个可选的参数。
|
||||
- `'strict'` - 当 `'strict'` 设置时，windowFunnel()仅对唯一值应用匹配条件。
|
||||
- `timestamp` — 包含时间的列。 数据类型支持： [日期](../../sql-reference/data-types/date.md), [日期时间](../../sql-reference/data-types/datetime.md#data_type-datetime) 和其他无符号整数类型（请注意，即使时间戳支持 `UInt64` 类型，它的值不能超过Int64最大值，即2^63-1）。
|
||||
- `cond` — 事件链的约束条件。 [UInt8](../../sql-reference/data-types/int-uint.md) 类型。
|
||||
|
||||
**返回值**
|
||||
|
||||
@ -284,7 +283,7 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN)
|
||||
└────────────┴─────────┴─────────────────────┴─────────┴─────────┘
|
||||
```
|
||||
|
||||
了解用户有多远 `user_id` 可以在2019的1-2月期间通过链条。
|
||||
了解用户`user_id` 可以在2019的1-2月期间通过链条多远。
|
||||
|
||||
查询:
|
||||
|
||||
@ -315,10 +314,10 @@ ORDER BY level ASC
|
||||
|
||||
## Retention {#retention}
|
||||
|
||||
该函数将一组条件作为参数，类型为1到32个参数 `UInt8` 表示事件是否满足特定条件。
|
||||
该函数将一组条件作为参数，类型为1到32个 `UInt8` 类型的参数，用来表示事件是否满足特定条件。
|
||||
任何条件都可以指定为参数（如 [WHERE](../../sql-reference/statements/select/where.md#select-where)).
|
||||
|
||||
除了第一个以外，条件成对适用：如果第一个和第二个是真的，第二个结果将是真的，如果第一个和fird是真的，第三个结果将是真的，等等。
|
||||
除了第一个以外，条件成对适用：如果第一个和第二个是真的，第二个结果将是真的，如果第一个和第三个是真的，第三个结果将是真的，等等。
|
||||
|
||||
**语法**
|
||||
|
||||
@ -328,22 +327,22 @@ retention(cond1, cond2, ..., cond32);
|
||||
|
||||
**参数**
|
||||
|
||||
- `cond` — an expression that returns a `UInt8` 结果（1或0）。
|
||||
- `cond` — 返回 `UInt8` 结果（1或0）的表达式。
|
||||
|
||||
**返回值**
|
||||
|
||||
数组为1或0。
|
||||
|
||||
- 1 — condition was met for the event.
|
||||
- 0 — condition wasn't met for the event.
|
||||
- 1 — 条件满足。
|
||||
- 0 — 条件不满足。
|
||||
|
||||
类型: `UInt8`.
|
||||
|
||||
**示例**
|
||||
|
||||
让我们考虑计算的一个例子 `retention` 功能，以确定网站流量。
|
||||
让我们考虑使用 `retention` 功能的一个例子 ，以确定网站流量。
|
||||
|
||||
**1.** Сreate a table to illustrate an example.
|
||||
**1.** 举例说明，先创建一张表。
|
||||
|
||||
``` sql
|
||||
CREATE TABLE retention_test(date Date, uid Int32) ENGINE = Memory;
|
||||
@ -402,7 +401,7 @@ SELECT * FROM retention_test
|
||||
└────────────┴─────┘
|
||||
```
|
||||
|
||||
**2.** 按唯一ID对用户进行分组 `uid` 使用 `retention` 功能。
|
||||
**2.** 按唯一ID `uid` 对用户进行分组，使用 `retention` 功能。
|
||||
|
||||
查询:
|
||||
|
||||
@ -466,7 +465,7 @@ FROM
|
||||
└────┴────┴────┘
|
||||
```
|
||||
|
||||
哪里:
|
||||
条件:
|
||||
|
||||
- `r1`-2020-01-01期间访问该网站的独立访问者数量（ `cond1` 条件）。
|
||||
- `r2`-在2020-01-01和2020-01-02之间的特定时间段内访问该网站的唯一访问者的数量 (`cond1` 和 `cond2` 条件）。
|
||||
@ -474,9 +473,9 @@ FROM
|
||||
|
||||
## uniqUpTo(N)(x) {#uniquptonx}
|
||||
|
||||
Calculates the number of different argument values if it is less than or equal to N. If the number of different argument values is greater than N, it returns N + 1.
|
||||
计算小于或者等于N的不同参数的个数。如果结果大于N，那返回N+1。
|
||||
|
||||
建议使用小Ns，高达10。 N的最大值为100。
|
||||
建议使用较小的Ns，比如：10。N的最大值为100。
|
||||
|
||||
对于聚合函数的状态，它使用的内存量等于1+N\*一个字节值的大小。
|
||||
对于字符串，它存储8个字节的非加密哈希。 也就是说，计算是近似的字符串。
|
||||
@ -488,12 +487,12 @@ Calculates the number of different argument values if it is less than or e
|
||||
用法示例:
|
||||
|
||||
``` text
|
||||
Problem: Generate a report that shows only keywords that produced at least 5 unique users.
|
||||
Solution: Write in the GROUP BY query SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5
|
||||
问题：产出一个不少于五个唯一用户的关键字报告
|
||||
解决方案： 写group by查询语句 HAVING uniqUpTo(4)(UserID) >= 5
|
||||
```
|
||||
|
||||
## sumMapFiltered(keys\_to\_keep)(keys, values) {#summapfilteredkeys-to-keepkeys-values}
|
||||
|
||||
和 [sumMap](reference.md#agg_functions-summap) 基本一致， 除了一个键数组作为参数传递。这在使用高基数key时尤其有用。
|
||||
|
||||
[原始文章](https://clickhouse.tech/docs/en/query_language/agg_functions/parametric_functions/) <!--hide-->
|
||||
|
||||
## sumMapFiltered(keys\_to\_keep)(键值) {#summapfilteredkeys-to-keepkeys-values}
|
||||
|
||||
同样的行为 [sumMap](reference.md#agg_functions-summap) 除了一个键数组作为参数传递。 这在使用高基数密钥时尤其有用。
|
||||
|
||||
@ -1,13 +1,11 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_priority: 36
|
||||
toc_title: "\u53C2\u8003\u8D44\u6599"
|
||||
toc_title: 聚合函数
|
||||
---
|
||||
|
||||
# 聚合函数引用 {#aggregate-functions-reference}
|
||||
|
||||
## 计数 {#agg_function-count}
|
||||
## count {#agg_function-count}
|
||||
|
||||
计数行数或非空值。
|
||||
|
||||
@ -73,7 +71,7 @@ SELECT count(DISTINCT num) FROM t
|
||||
|
||||
这个例子表明 `count(DISTINCT num)` 由执行 `uniqExact` 根据功能 `count_distinct_implementation` 设定值。
|
||||
|
||||
## 任何(x) {#agg_function-any}
|
||||
## any(x) {#agg_function-any}
|
||||
|
||||
选择第一个遇到的值。
|
||||
查询可以以任何顺序执行，甚至每次都以不同的顺序执行，因此此函数的结果是不确定的。
|
||||
@ -115,7 +113,7 @@ FROM ontime
|
||||
选择遇到的最后一个值。
|
||||
其结果是一样不确定的 `any` 功能。
|
||||
|
||||
## 集团比特 {#groupbitand}
|
||||
## groupBitAnd {#groupbitand}
|
||||
|
||||
按位应用 `AND` 对于一系列的数字。
|
||||
|
||||
@ -337,7 +335,7 @@ SELECT argMin(user, salary) FROM salary
|
||||
总计 ‘value’ 数组根据在指定的键 ‘key’ 阵列。
|
||||
传递键和值数组的元组与传递两个键和值数组是同义的。
|
||||
元素的数量 ‘key’ 和 ‘value’ 总计的每一行必须相同。
|
||||
Returns a tuple of two arrays: keys in sorted order, and values summed for the corresponding keys.
|
||||
返回两个数组的一个二元组： key是排好序的，value是对应key的求和。
|
||||
|
||||
示例:
|
||||
|
||||
@ -374,7 +372,7 @@ GROUP BY timeslot
|
||||
|
||||
## skewPop {#skewpop}
|
||||
|
||||
计算 [歪斜](https://en.wikipedia.org/wiki/Skewness) 的序列。
|
||||
计算的序列[偏度](https://en.wikipedia.org/wiki/Skewness)。
|
||||
|
||||
``` sql
|
||||
skewPop(expr)
|
||||
@ -386,7 +384,7 @@ skewPop(expr)
|
||||
|
||||
**返回值**
|
||||
|
||||
The skewness of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md)
|
||||
给定序列的偏度。类型 — [Float64](../../sql-reference/data-types/float.md)
|
||||
|
||||
**示例**
|
||||
|
||||
@ -410,7 +408,7 @@ skewSamp(expr)
|
||||
|
||||
**返回值**
|
||||
|
||||
The skewness of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). 如果 `n <= 1` (`n` 是样本的大小），则该函数返回 `nan`.
|
||||
给定序列的偏度。 类型 — [Float64](../../sql-reference/data-types/float.md). 如果 `n <= 1` (`n` 是样本的大小），则该函数返回 `nan`.
|
||||
|
||||
**示例**
|
||||
|
||||
@ -432,7 +430,7 @@ kurtPop(expr)
|
||||
|
||||
**返回值**
|
||||
|
||||
The kurtosis of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md)
|
||||
给定序列的峰度。 类型 — [Float64](../../sql-reference/data-types/float.md)
|
||||
|
||||
**示例**
|
||||
|
||||
@ -456,7 +454,7 @@ kurtSamp(expr)
|
||||
|
||||
**返回值**
|
||||
|
||||
The kurtosis of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). 如果 `n <= 1` (`n` 是样本的大小），则该函数返回 `nan`.
|
||||
给定序列的峰度。类型 — [Float64](../../sql-reference/data-types/float.md). 如果 `n <= 1` (`n` 是样本的大小），则该函数返回 `nan`.
|
||||
|
||||
**示例**
|
||||
|
||||
@ -533,7 +531,7 @@ FROM (
|
||||
只适用于数字。
|
||||
结果总是Float64。
|
||||
|
||||
## 平均加权 {#avgweighted}
|
||||
## avgWeighted {#avgweighted}
|
||||
|
||||
计算 [加权算术平均值](https://en.wikipedia.org/wiki/Weighted_arithmetic_mean).
|
||||
|
||||
@ -545,10 +543,10 @@ avgWeighted(x, weight)
|
||||
|
||||
**参数**
|
||||
|
||||
- `x` — Values. [整数](../data-types/int-uint.md) 或 [浮点](../data-types/float.md).
|
||||
- `weight` — Weights of the values. [整数](../data-types/int-uint.md) 或 [浮点](../data-types/float.md).
|
||||
- `x` — 值。 [整数](../data-types/int-uint.md) 或 [浮点](../data-types/float.md).
|
||||
- `weight` — 值的加权。 [整数](../data-types/int-uint.md) 或 [浮点](../data-types/float.md).
|
||||
|
||||
类型 `x` 和 `weight` 一定是一样的
|
||||
`x` 和 `weight` 的类型一定是一样的
|
||||
|
||||
**返回值**
|
||||
|
||||
@ -590,7 +588,7 @@ uniq(x[, ...])
|
||||
|
||||
- A [UInt64](../../sql-reference/data-types/int-uint.md)-键入号码。
|
||||
|
||||
**实施细节**
|
||||
**实现细节**
|
||||
|
||||
功能:
|
||||
|
||||
@ -598,7 +596,7 @@ uniq(x[, ...])
|
||||
|
||||
- 使用自适应采样算法。 对于计算状态，该函数使用最多65536个元素哈希值的样本。
|
||||
|
||||
This algorithm is very accurate and very efficient on the CPU. When the query contains several of these functions, using `uniq` is almost as fast as using other aggregate functions.
|
||||
这个算法是非常精确的，并且对于CPU来说非常高效。如果查询包含一些这样的函数，那和其他聚合函数相比 `uniq` 将是几乎一样快。
|
||||
|
||||
- 确定性地提供结果（它不依赖于查询处理顺序）。
|
||||
|
||||
@ -629,17 +627,17 @@ uniqCombined(HLL_precision)(x[, ...])
|
||||
|
||||
**返回值**
|
||||
|
||||
- 一个数字 [UInt64](../../sql-reference/data-types/int-uint.md)-键入号码。
|
||||
- 一个[UInt64](../../sql-reference/data-types/int-uint.md)类型的数字。
|
||||
|
||||
**实施细节**
|
||||
**实现细节**
|
||||
|
||||
功能:
|
||||
|
||||
- 计算散列（64位散列 `String` 否则32位）对于聚合中的所有参数，然后在计算中使用它。
|
||||
|
||||
- 使用三种算法的组合：数组、哈希表和HyperLogLog与error错表。
|
||||
- 使用三种算法的组合：数组、哈希表和包含错误修正表的HyperLogLog。
|
||||
|
||||
For a small number of distinct elements, an array is used. When the set size is larger, a hash table is used. For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory.
|
||||
少量的不同的值，使用数组。 值再多一些，使用哈希表。对于大量的数据来说，使用HyperLogLog，HyperLogLog占用一个固定的内存空间。
|
||||
|
||||
- 确定性地提供结果（它不依赖于查询处理顺序）。
|
||||
|
||||
@ -650,7 +648,7 @@ uniqCombined(HLL_precision)(x[, ...])
|
||||
|
||||
- 消耗少几倍的内存。
|
||||
- 计算精度高出几倍。
|
||||
- 通常具有略低的性能。 在某些情况下, `uniqCombined` 可以表现得比 `uniq`，例如，使用通过网络传输大量聚合状态的分布式查询。
|
||||
- 通常具有略低的性能。 在某些情况下, `uniqCombined` 可以表现得比 `uniq` 好，例如，使用通过网络传输大量聚合状态的分布式查询。
|
||||
|
||||
**另请参阅**
|
||||
|
||||
@ -679,7 +677,7 @@ uniqHLL12(x[, ...])
|
||||
|
||||
- A [UInt64](../../sql-reference/data-types/int-uint.md)-键入号码。
|
||||
|
||||
**实施细节**
|
||||
**实现细节**
|
||||
|
||||
功能:
|
||||
|
||||
@ -707,9 +705,9 @@ uniqHLL12(x[, ...])
|
||||
uniqExact(x[, ...])
|
||||
```
|
||||
|
||||
使用 `uniqExact` 功能，如果你绝对需要一个确切的结果。 否则使用 [uniq](#agg_function-uniq) 功能。
|
||||
如果你绝对需要一个确切的结果，使用 `uniqExact` 功能。 否则使用 [uniq](#agg_function-uniq) 功能。
|
||||
|
||||
该 `uniqExact` 功能使用更多的内存比 `uniq`，因为状态的大小随着不同值的数量的增加而无界增长。
|
||||
`uniqExact` 比 `uniq` 使用更多的内存，因为状态的大小随着不同值的数量的增加而无界增长。
|
||||
|
||||
**参数**
|
||||
|
||||
@ -721,7 +719,7 @@ uniqExact(x[, ...])
|
||||
- [uniqCombined](#agg_function-uniqcombined)
|
||||
- [uniqHLL12](#agg_function-uniqhll12)
|
||||
|
||||
## 群交(x),群交(max\_size)(x) {#agg_function-grouparray}
|
||||
## groupArray(x), groupArray(max\_size)(x) {#agg_function-grouparray}
|
||||
|
||||
创建参数值的数组。
|
||||
值可以按任何（不确定）顺序添加到数组中。
|
||||
@ -748,10 +746,10 @@ groupArrayInsertAt(default_x, size)(x, pos);
|
||||
|
||||
**参数**
|
||||
|
||||
- `x` — Value to be inserted. [表达式](../syntax.md#syntax-expressions) 导致的一个 [支持的数据类型](../../sql-reference/data-types/index.md).
|
||||
- `pos` — Position at which the specified element `x` 将被插入。 数组中的索引编号从零开始。 [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges).
|
||||
- `default_x`— Default value for substituting in empty positions. Optional parameter. [表达式](../syntax.md#syntax-expressions) 导致为配置的数据类型 `x` 参数。 如果 `default_x` 未定义，则 [默认值](../../sql-reference/statements/create.md#create-default-values) 被使用。
|
||||
- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value `default_x` 必须指定。 [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges).
|
||||
- `x` — 被插入的值。[表达式](../syntax.md#syntax-expressions) 导致的一个 [支持的数据类型](../../sql-reference/data-types/index.md).
|
||||
- `pos` — `x` 将被插入的位置。 数组中的索引编号从零开始。 [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges).
|
||||
- `default_x`— 如果代入值为空，则使用默认值。可选参数。[表达式](../syntax.md#syntax-expressions) 为 `x` 数据类型的数据。 如果 `default_x` 未定义，则 [默认值](../../sql-reference/statements/create.md#create-default-values) 被使用。
|
||||
- `size`— 结果数组的长度。可选参数。如果使用该参数，`default_x` 必须指定。 [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges).
|
||||
|
||||
**返回值**
|
||||
|
||||
@ -803,7 +801,7 @@ SELECT groupArrayInsertAt('-', 5)(toString(number), number * 2) FROM numbers(5);
|
||||
└───────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
元件的多线程插入到一个位置。
|
||||
在一个位置多线程插入数据。
|
||||
|
||||
查询:
|
||||
|
||||
@ -832,8 +830,8 @@ groupArrayMovingSum(window_size)(numbers_for_summing)
|
||||
|
||||
**参数**
|
||||
|
||||
- `numbers_for_summing` — [表达式](../syntax.md#syntax-expressions) 生成数值数据类型值。
|
||||
- `window_size` — Size of the calculation window.
|
||||
- `numbers_for_summing` — [表达式](../syntax.md#syntax-expressions) 为数值数据类型值。
|
||||
- `window_size` — 窗口大小。
|
||||
|
||||
**返回值**
|
||||
|
||||
@ -906,13 +904,13 @@ groupArrayMovingAvg(window_size)(numbers_for_summing)
|
||||
**参数**
|
||||
|
||||
- `numbers_for_summing` — [表达式](../syntax.md#syntax-expressions) 生成数值数据类型值。
|
||||
- `window_size` — Size of the calculation window.
|
||||
- `window_size` — 窗口大小。
|
||||
|
||||
**返回值**
|
||||
|
||||
- 与输入数据大小和类型相同的数组。
|
||||
|
||||
该函数使用 [四舍五入到零](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero). 它截断结果数据类型的小数位数。
|
||||
该函数使用 [四舍五入到零](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero). 它截断无意义的小数位来保证结果的数据类型。
|
||||
|
||||
**示例**
|
||||
|
||||
@ -967,20 +965,20 @@ FROM t
|
||||
└───────────┴──────────────────────────────────┴───────────────────────┘
|
||||
```
|
||||
|
||||
## 禄,赂麓ta脌麓,):脡,,拢脢,group媒group)galaxy s8碌胫脢)禄煤)酶脱脩) {#groupuniqarrayx-groupuniqarraymax-sizex}
|
||||
## groupUniqArray(x), groupUniqArray(max\_size)(x) {#groupuniqarrayx-groupuniqarraymax-sizex}
|
||||
|
||||
从不同的参数值创建一个数组。 内存消耗是一样的 `uniqExact` 功能。
|
||||
|
||||
第二个版本（与 `max_size` 参数）将结果数组的大小限制为 `max_size` 元素。
|
||||
第二个版本（`max_size` 参数）将结果数组的大小限制为 `max_size` 元素。
|
||||
例如, `groupUniqArray(1)(x)` 相当于 `[any(x)]`.
|
||||
|
||||
## 分位数 {#quantile}
|
||||
## quantile {#quantile}
|
||||
|
||||
计算近似值 [分位数](https://en.wikipedia.org/wiki/Quantile) 的数字数据序列。
|
||||
计算数字序列的近似[分位数](https://en.wikipedia.org/wiki/Quantile)。
|
||||
|
||||
此功能适用 [油藏采样](https://en.wikipedia.org/wiki/Reservoir_sampling) 随着储存器大小高达8192和随机数发生器进行采样。 结果是非确定性的。 要获得精确的分位数，请使用 [quantileExact](#quantileexact) 功能。
|
||||
此功能适用 [水塘抽样(](https://en.wikipedia.org/wiki/Reservoir_sampling)，使用储存器最大到8192和随机数发生器进行采样。 结果是非确定性的。 要获得精确的分位数，请使用 [quantileExact](#quantileexact) 功能。
|
||||
|
||||
当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。
|
||||
当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。
|
||||
|
||||
**语法**
|
||||
|
||||
@ -992,12 +990,12 @@ quantile(level)(expr)
|
||||
|
||||
**参数**
|
||||
|
||||
- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md).
|
||||
- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。
|
||||
|
||||
**返回值**
|
||||
|
||||
- 指定电平的近似分位数。
|
||||
- 指定层次的近似分位数。
|
||||
|
||||
类型:
|
||||
|
||||
@ -1037,13 +1035,13 @@ SELECT quantile(val) FROM t
|
||||
- [中位数](#median)
|
||||
- [分位数](#quantiles)
|
||||
|
||||
## 量化确定 {#quantiledeterministic}
|
||||
## quantileDeterministic {#quantiledeterministic}
|
||||
|
||||
计算近似值 [分位数](https://en.wikipedia.org/wiki/Quantile) 的数字数据序列。
|
||||
计算数字序列的近似[分位数](https://en.wikipedia.org/wiki/Quantile)。
|
||||
|
||||
此功能适用 [油藏采样](https://en.wikipedia.org/wiki/Reservoir_sampling) 与储层大小高达8192和采样的确定性算法。 结果是确定性的。 要获得精确的分位数，请使用 [quantileExact](#quantileexact) 功能。
|
||||
此功能适用 [水塘抽样(](https://en.wikipedia.org/wiki/Reservoir_sampling)，使用储存器最大到8192和随机数发生器进行采样。 结果是非确定性的。 要获得精确的分位数，请使用 [quantileExact](#quantileexact) 功能。
|
||||
|
||||
当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。
|
||||
当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。
|
||||
|
||||
**语法**
|
||||
|
||||
@ -1055,13 +1053,13 @@ quantileDeterministic(level)(expr, determinator)
|
||||
|
||||
**参数**
|
||||
|
||||
- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md).
|
||||
- `determinator` — Number whose hash is used instead of a random number generator in the reservoir sampling algorithm to make the result of sampling deterministic. As a determinator you can use any deterministic positive number, for example, a user id or an event id. If the same determinator value occures too often, the function works incorrectly.
|
||||
- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。
|
||||
- `determinator` — 一个数字，其hash被用来代替在水塘抽样中随机生成的数字，这样可以保证取样的确定性。你可以使用用户ID或者事件ID等任何正数，但是如果相同的 `determinator` 出现多次，那结果很可能不正确。
|
||||
|
||||
**返回值**
|
||||
|
||||
- 指定电平的近似分位数。
|
||||
- 指定层次的近似分位数。
|
||||
|
||||
类型:
|
||||
|
||||
@ -1103,11 +1101,11 @@ SELECT quantileDeterministic(val, 1) FROM t
|
||||
|
||||
## quantileExact {#quantileexact}
|
||||
|
||||
正是计算 [分位数](https://en.wikipedia.org/wiki/Quantile) 的数字数据序列。
|
||||
准确计算数字序列的[分位数](https://en.wikipedia.org/wiki/Quantile)。
|
||||
|
||||
To get exact value, all the passed values are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` 内存，其中 `n` 是传递的多个值。 然而，对于少量的值，该函数是非常有效的。
|
||||
为了准确计算，所有输入的数据被合并为一个数组，并且部分的排序。因此该函数需要 `O(n)` 的内存，n为输入数据的个数。但是对于少量数据来说，该函数还是非常有效的。
|
||||
|
||||
当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。
|
||||
当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。
|
||||
|
||||
**语法**
|
||||
|
||||
@ -1119,12 +1117,12 @@ quantileExact(level)(expr)
|
||||
|
||||
**参数**
|
||||
|
||||
- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md).
|
||||
- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。
|
||||
|
||||
**返回值**
|
||||
|
||||
- 指定电平的分位数。
|
||||
- 指定层次的分位数。
|
||||
|
||||
类型:
|
||||
|
||||
@ -1153,13 +1151,13 @@ SELECT quantileExact(number) FROM numbers(10)
|
||||
- [中位数](#median)
|
||||
- [分位数](#quantiles)
|
||||
|
||||
## 分位数加权 {#quantileexactweighted}
|
||||
## quantileExactWeighted {#quantileexactweighted}
|
||||
|
||||
正是计算 [分位数](https://en.wikipedia.org/wiki/Quantile) 数值数据序列，考虑到每个元素的权重。
|
||||
考虑到每个元素的权重，然后准确计算数值序列的[分位数](https://en.wikipedia.org/wiki/Quantile)。
|
||||
|
||||
To get exact value, all the passed values are combined into an array, which is then partially sorted. Each value is counted with its weight, as if it is present `weight` times. A hash table is used in the algorithm. Because of this, if the passed values are frequently repeated, the function consumes less RAM than [quantileExact](#quantileexact). 您可以使用此功能，而不是 `quantileExact` 并指定重量1。
|
||||
为了准确计算，所有输入的数据被合并为一个数组，并且部分的排序。每个输入值需要根据 `weight` 计算求和。该算法使用哈希表。正因为如此，在数据重复较多的时候使用的内存是少于[quantileExact](#quantileexact)的。 您可以使用此函数代替 `quantileExact` 并指定重量1。
|
||||
|
||||
当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。
|
||||
当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。
|
||||
|
||||
**语法**
|
||||
|
||||
@ -1171,13 +1169,13 @@ quantileExactWeighted(level)(expr, weight)
|
||||
|
||||
**参数**
|
||||
|
||||
- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md).
|
||||
- `weight` — Column with weights of sequence members. Weight is a number of value occurrences.
|
||||
- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。
|
||||
- `weight` — 权重序列。 权重是一个数据出现的数值。
|
||||
|
||||
**返回值**
|
||||
|
||||
- 指定电平的分位数。
|
||||
- 指定层次的分位数。
|
||||
|
||||
类型:
|
||||
|
||||
@ -1217,13 +1215,13 @@ SELECT quantileExactWeighted(n, val) FROM t
|
||||
- [中位数](#median)
|
||||
- [分位数](#quantiles)
|
||||
|
||||
## 分位定时 {#quantiletiming}
|
||||
## quantileTiming {#quantiletiming}
|
||||
|
||||
随着确定的精度计算 [分位数](https://en.wikipedia.org/wiki/Quantile) 的数字数据序列。
|
||||
使用确定的精度计算数字数据序列的[分位数](https://en.wikipedia.org/wiki/Quantile)。
|
||||
|
||||
结果是确定性的（它不依赖于查询处理顺序）。 该函数针对描述加载网页时间或后端响应时间等分布的序列进行了优化。
|
||||
|
||||
当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。
|
||||
当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。
|
||||
|
||||
**语法**
|
||||
|
||||
@ -1235,12 +1233,12 @@ quantileTiming(level)(expr)
|
||||
|
||||
**参数**
|
||||
|
||||
- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
|
||||
- `expr` — [表达式](../syntax.md#syntax-expressions) 在一个列值返回 [浮动\*](../../sql-reference/data-types/float.md)-键入号码。
|
||||
- `expr` — [表达式](../syntax.md#syntax-expressions)，返回 [浮动\*](../../sql-reference/data-types/float.md)类型数据。
|
||||
|
||||
- If negative values are passed to the function, the behavior is undefined.
|
||||
- If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000.
|
||||
- 如果输入负值，那结果是不可预期的。
|
||||
- 如果输入值大于30000（页面加载时间大于30s），那我们假设为30000。
|
||||
|
||||
**精度**
|
||||
|
||||
@ -1252,16 +1250,16 @@ quantileTiming(level)(expr)
|
||||
否则，计算结果将四舍五入到16毫秒的最接近倍数。
|
||||
|
||||
!!! note "注"
|
||||
对于计算页面加载时间分位数，此函数比 [分位数](#quantile).
|
||||
对于计算页面加载时间分位数，此函数比 [分位数](#quantile)更有效和准确。
|
||||
|
||||
**返回值**
|
||||
|
||||
- 指定电平的分位数。
|
||||
- 指定层次的分位数。
|
||||
|
||||
类型: `Float32`.
|
||||
|
||||
!!! note "注"
|
||||
如果没有值传递给函数（当使用 `quantileTimingIf`), [阿南](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 被返回。 这样做的目的是将这些案例与导致零的案例区分开来。 看 [按条款订购](../statements/select/order-by.md#select-order-by) 对于排序注意事项 `NaN` 值。
|
||||
如果没有值传递给函数（当使用 `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 被返回。 这样做的目的是将这些案例与导致零的案例区分开来。 看 [ORDER BY clause](../statements/select/order-by.md#select-order-by) 对于 `NaN` 值排序注意事项。
|
||||
|
||||
**示例**
|
||||
|
||||
@ -1300,13 +1298,13 @@ SELECT quantileTiming(response_time) FROM t
|
||||
- [中位数](#median)
|
||||
- [分位数](#quantiles)
|
||||
|
||||
## 分位时间加权 {#quantiletimingweighted}
|
||||
## quantileTimingWeighted {#quantiletimingweighted}
|
||||
|
||||
随着确定的精度计算 [分位数](https://en.wikipedia.org/wiki/Quantile) 根据每个序列成员的权重对数字数据序列进行处理。
|
||||
根据每个序列成员的权重，使用确定的精度计算数字序列的[分位数](https://en.wikipedia.org/wiki/Quantile)。
|
||||
|
||||
结果是确定性的（它不依赖于查询处理顺序）。 该函数针对描述加载网页时间或后端响应时间等分布的序列进行了优化。
|
||||
|
||||
当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。
|
||||
当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。
|
||||
|
||||
**语法**
|
||||
|
||||
@ -1318,14 +1316,14 @@ quantileTimingWeighted(level)(expr, weight)
|
||||
|
||||
**参数**
|
||||
|
||||
- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
|
||||
- `expr` — [表达式](../syntax.md#syntax-expressions) 在一个列值返回 [浮动\*](../../sql-reference/data-types/float.md)-键入号码。
|
||||
- `expr` — [表达式](../syntax.md#syntax-expressions)，返回 [浮动\*](../../sql-reference/data-types/float.md)类型数据。
|
||||
|
||||
- If negative values are passed to the function, the behavior is undefined.
|
||||
- If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000.
|
||||
- 如果输入负值，那结果是不可预期的。
|
||||
- 如果输入值大于30000（页面加载时间大于30s），那我们假设为30000。
|
||||
|
||||
- `weight` — Column with weights of sequence elements. Weight is a number of value occurrences.
|
||||
- `weight` — 权重序列。 权重是一个数据出现的数值。
|
||||
|
||||
**精度**
|
||||
|
||||
@ -1337,16 +1335,16 @@ quantileTimingWeighted(level)(expr, weight)
|
||||
否则，计算结果将四舍五入到16毫秒的最接近倍数。
|
||||
|
||||
!!! note "注"
|
||||
对于计算页面加载时间分位数，此函数比 [分位数](#quantile).
|
||||
对于计算页面加载时间分位数，此函数比 [分位数](#quantile)更高效和准确。
|
||||
|
||||
**返回值**
|
||||
|
||||
- 指定电平的分位数。
|
||||
- 指定层次的分位数。
|
||||
|
||||
类型: `Float32`.
|
||||
|
||||
!!! note "注"
|
||||
如果没有值传递给函数（当使用 `quantileTimingIf`), [阿南](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 被返回。 这样做的目的是将这些案例与导致零的案例区分开来。 看 [按条款订购](../statements/select/order-by.md#select-order-by) 对于排序注意事项 `NaN` 值。
|
||||
如果没有值传递给函数（当使用 `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 被返回。 这样做的目的是将这些案例与导致零的案例区分开来。看 [ORDER BY clause](../statements/select/order-by.md#select-order-by) 对于 `NaN` 值排序注意事项。
|
||||
|
||||
**示例**
|
||||
|
||||
@ -1384,13 +1382,13 @@ SELECT quantileTimingWeighted(response_time, weight) FROM t
|
||||
|
||||
## quantileTDigest {#quantiletdigest}
|
||||
|
||||
计算近似值 [分位数](https://en.wikipedia.org/wiki/Quantile) 使用的数字数据序列 [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) 算法。
|
||||
使用[t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) 算法计算近似[分位数](https://en.wikipedia.org/wiki/Quantile)。
|
||||
|
||||
最大误差为1%。 内存消耗 `log(n)`，哪里 `n` 是多个值。 结果取决于运行查询的顺序，并且是不确定的。
|
||||
最大误差为1%。 内存消耗 `log(n)`，这里 `n` 是值的个数。 结果取决于运行查询的顺序，并且是不确定的。
|
||||
|
||||
该功能的性能低于性能 [分位数](#quantile) 或 [分位定时](#quantiletiming). 在状态大小与精度的比率方面，这个函数比 `quantile`.
|
||||
该功能的性能低于性能 [分位数](#quantile) 或 [时间分位](#quantiletiming). 在状态大小与精度的比率方面，这个函数比 `quantile`更优秀。
|
||||
|
||||
当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。
|
||||
当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。
|
||||
|
||||
**语法**
|
||||
|
||||
@ -1402,12 +1400,12 @@ quantileTDigest(level)(expr)
|
||||
|
||||
**参数**
|
||||
|
||||
- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md).
|
||||
- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。
|
||||
|
||||
**回值**
|
||||
|
||||
- 指定电平的近似分位数。
|
||||
- 指定层次的分位数。
|
||||
|
||||
类型:
|
||||
|
||||
@ -1438,13 +1436,13 @@ SELECT quantileTDigest(number) FROM numbers(10)
|
||||
|
||||
## quantileTDigestWeighted {#quantiletdigestweighted}
|
||||
|
||||
计算近似值 [分位数](https://en.wikipedia.org/wiki/Quantile) 使用的数字数据序列 [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) 算法。 该函数考虑了每个序列成员的权重。 最大误差为1%。 内存消耗 `log(n)`，哪里 `n` 是多个值。
|
||||
使用[t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) 算法计算近似[分位数](https://en.wikipedia.org/wiki/Quantile)。 该函数考虑了每个序列成员的权重。最大误差为1%。 内存消耗 `log(n)`，这里 `n` 是值的个数。
|
||||
|
||||
该功能的性能低于性能 [分位数](#quantile) 或 [分位定时](#quantiletiming). 在状态大小与精度的比率方面，这个函数比 `quantile`.
|
||||
该功能的性能低于性能 [分位数](#quantile) 或 [时间分位](#quantiletiming). 在状态大小与精度的比率方面，这个函数比 `quantile`更优秀。
|
||||
|
||||
结果取决于运行查询的顺序，并且是不确定的。
|
||||
|
||||
当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。
|
||||
当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能
|
||||
|
||||
**语法**
|
||||
|
||||
@ -1456,13 +1454,13 @@ quantileTDigest(level)(expr)
|
||||
|
||||
**参数**
|
||||
|
||||
- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md).
|
||||
- `weight` — Column with weights of sequence elements. Weight is a number of value occurrences.
|
||||
- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。
|
||||
- `weight` — 权重序列。 权重是一个数据出现的数值。
|
||||
|
||||
**返回值**
|
||||
|
||||
- 指定电平的近似分位数。
|
||||
- 指定层次的分位数。
|
||||
|
||||
类型:
|
||||
|
||||
@ -1491,20 +1489,20 @@ SELECT quantileTDigestWeighted(number, 1) FROM numbers(10)
|
||||
- [中位数](#median)
|
||||
- [分位数](#quantiles)
|
||||
|
||||
## 中位数 {#median}
|
||||
## median {#median}
|
||||
|
||||
该 `median*` 函数是相应的别名 `quantile*` 功能。 它们计算数字数据样本的中位数。
|
||||
`median*` 函数是 `quantile*` 函数的别名。 它们计算数字数据样本的中位数。
|
||||
|
||||
功能:
|
||||
函数:
|
||||
|
||||
- `median` — Alias for [分位数](#quantile).
|
||||
- `medianDeterministic` — Alias for [量化确定](#quantiledeterministic).
|
||||
- `medianExact` — Alias for [quantileExact](#quantileexact).
|
||||
- `medianExactWeighted` — Alias for [分位数加权](#quantileexactweighted).
|
||||
- `medianTiming` — Alias for [分位定时](#quantiletiming).
|
||||
- `medianTimingWeighted` — Alias for [分位时间加权](#quantiletimingweighted).
|
||||
- `medianTDigest` — Alias for [quantileTDigest](#quantiletdigest).
|
||||
- `medianTDigestWeighted` — Alias for [quantileTDigestWeighted](#quantiletdigestweighted).
|
||||
- `median` — [quantile](#quantile)别名。
|
||||
- `medianDeterministic` — [quantileDeterministic](#quantiledeterministic)别名。
|
||||
- `medianExact` — [quantileExact](#quantileexact)别名。
|
||||
- `medianExactWeighted` — [quantileExactWeighted](#quantileexactweighted)别名。
|
||||
- `medianTiming` — [quantileTiming](#quantiletiming)别名。
|
||||
- `medianTimingWeighted` — [quantileTimingWeighted](#quantiletimingweighted)别名。
|
||||
- `medianTDigest` — [quantileTDigest](#quantiletdigest)别名。
|
||||
- `medianTDigestWeighted` — [quantileTDigestWeighted](#quantiletdigestweighted)别名。
|
||||
|
||||
**示例**
|
||||
|
||||
@ -1535,11 +1533,11 @@ SELECT medianDeterministic(val, 1) FROM t
|
||||
|
||||
## quantiles(level1, level2, …)(x) {#quantiles}
|
||||
|
||||
所有分位数函数也具有相应的分位数函数: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. 这些函数在一遍中计算所列电平的所有分位数，并返回结果值的数组。
|
||||
所有分位数函数也有相应的函数: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`。这些函数一次计算所列层次的所有分位数，并返回结果值的数组。
|
||||
|
||||
## varSamp(x) {#varsampx}
|
||||
|
||||
计算金额 `Σ((x - x̅)^2) / (n - 1)`，哪里 `n` 是样本大小和 `x̅`是平均值 `x`.
|
||||
计算 `Σ((x - x̅)^2) / (n - 1)`，这里 `n` 是样本大小， `x̅`是`x`的平均值。
|
||||
|
||||
它表示随机变量的方差的无偏估计，如果传递的值形成其样本。
|
||||
|
||||
@ -1550,23 +1548,23 @@ SELECT medianDeterministic(val, 1) FROM t
|
||||
|
||||
## varPop(x) {#varpopx}
|
||||
|
||||
计算金额 `Σ((x - x̅)^2) / n`，哪里 `n` 是样本大小和 `x̅`是平均值 `x`.
|
||||
计算 `Σ((x - x̅)^2) / n`，这里 `n` 是样本大小， `x̅`是`x`的平均值。
|
||||
|
||||
换句话说，分散为一组值。 返回 `Float64`.
|
||||
换句话说，计算一组数据的离差。 返回 `Float64`。
|
||||
|
||||
!!! note "注"
|
||||
该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `varPopStable` 功能。 它的工作速度较慢，但提供较低的计算错误。
|
||||
|
||||
## stddevSamp(x) {#stddevsampx}
|
||||
|
||||
结果等于平方根 `varSamp(x)`.
|
||||
结果等于平方根 `varSamp(x)`。
|
||||
|
||||
!!! note "注"
|
||||
该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `stddevSampStable` 功能。 它的工作速度较慢，但提供较低的计算错误。
|
||||
|
||||
## stddevPop(x) {#stddevpopx}
|
||||
|
||||
结果等于平方根 `varPop(x)`.
|
||||
结果等于平方根 `varPop(x)`。
|
||||
|
||||
!!! note "注"
|
||||
该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `stddevPopStable` 功能。 它的工作速度较慢，但提供较低的计算错误。
|
||||
@ -1575,15 +1573,15 @@ SELECT medianDeterministic(val, 1) FROM t
|
||||
|
||||
返回指定列中近似最常见值的数组。 生成的数组按值的近似频率降序排序（而不是值本身）。
|
||||
|
||||
实现了 [过滤节省空间](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) 基于reduce-and-combine算法的TopK分析算法 [并行节省空间](https://arxiv.org/pdf/1401.0702.pdf).
|
||||
实现了[过滤节省空间](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf)算法， 使用基于reduce-and-combine的算法，借鉴[并行节省空间](https://arxiv.org/pdf/1401.0702.pdf).
|
||||
|
||||
``` sql
|
||||
topK(N)(column)
|
||||
```
|
||||
|
||||
此函数不提供保证的结果。 在某些情况下，可能会发生错误，并且可能会返回不是最常见值的常见值。
|
||||
此函数不提供保证的结果。 在某些情况下，可能会发生错误，并且可能会返回不是最高频的值。
|
||||
|
||||
我们建议使用 `N < 10` 值;性能降低了大 `N` 值。 的最大值 `N = 65536`.
|
||||
我们建议使用 `N < 10` 值，`N` 值越大，性能越低。最大值 `N = 65536`。
|
||||
|
||||
**参数**
|
||||
|
||||
@ -1593,11 +1591,11 @@ topK(N)(column)
|
||||
|
||||
**参数**
|
||||
|
||||
- ' x ' – The value to calculate frequency.
|
||||
- ' x ' – 计算的频率值。
|
||||
|
||||
**示例**
|
||||
|
||||
就拿 [时间](../../getting-started/example-datasets/ontime.md) 数据集，并选择在三个最频繁出现的值 `AirlineID` 列。
|
||||
就拿 [OnTime](../../getting-started/example-datasets/ontime.md) 数据集来说，选择`AirlineID` 列中出现最频繁的三个。
|
||||
|
||||
``` sql
|
||||
SELECT topK(3)(AirlineID) AS res
|
||||
@ -1612,7 +1610,7 @@ FROM ontime
|
||||
|
||||
## topKWeighted {#topkweighted}
|
||||
|
||||
类似于 `topK` 但需要一个整数类型的附加参数 - `weight`. 每个价值都被记入 `weight` 次频率计算。
|
||||
类似于 `topK` 但需要一个整数类型的附加参数 - `weight`. 每个输入都被记入 `weight` 次频率计算。
|
||||
|
||||
**语法**
|
||||
|
||||
@ -1622,12 +1620,12 @@ topKWeighted(N)(x, weight)
|
||||
|
||||
**参数**
|
||||
|
||||
- `N` — The number of elements to return.
|
||||
- `N` — 返回值个数。
|
||||
|
||||
**参数**
|
||||
|
||||
- `x` – The value.
|
||||
- `weight` — The weight. [UInt8](../../sql-reference/data-types/int-uint.md).
|
||||
- `x` – 输入值。
|
||||
- `weight` — 权重。 [UInt8](../../sql-reference/data-types/int-uint.md)类型。
|
||||
|
||||
**返回值**
|
||||
|
||||
@ -1651,36 +1649,36 @@ SELECT topKWeighted(10)(number, number) FROM numbers(1000)
|
||||
|
||||
## covarSamp(x,y) {#covarsampx-y}
|
||||
|
||||
计算的值 `Σ((x - x̅)(y - y̅)) / (n - 1)`.
|
||||
计算 `Σ((x - x̅)(y - y̅)) / (n - 1)`。
|
||||
|
||||
返回Float64。 当 `n <= 1`, returns +∞.
|
||||
返回Float64。 当 `n <= 1`, returns +∞。
|
||||
|
||||
!!! note "注"
|
||||
该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `covarSampStable` 功能。 它的工作速度较慢，但提供较低的计算错误。
|
||||
|
||||
## covarPop(x,y) {#covarpopx-y}
|
||||
|
||||
计算的值 `Σ((x - x̅)(y - y̅)) / n`.
|
||||
计算 `Σ((x - x̅)(y - y̅)) / n`。
|
||||
|
||||
!!! note "注"
|
||||
该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `covarPopStable` 功能。 它的工作速度较慢，但提供了较低的计算错误。
|
||||
|
||||
## corr(x,y) {#corrx-y}
|
||||
|
||||
计算Pearson相关系数: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`.
|
||||
计算Pearson相关系数: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`。
|
||||
|
||||
!!! note "注"
|
||||
该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `corrStable` 功能。 它的工作速度较慢，但提供较低的计算错误。
|
||||
|
||||
## categoricalInformationValue {#categoricalinformationvalue}
|
||||
|
||||
计算的值 `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` 对于每个类别。
|
||||
对于每个类别计算 `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` 。
|
||||
|
||||
``` sql
|
||||
categoricalInformationValue(category1, category2, ..., tag)
|
||||
```
|
||||
|
||||
结果指示离散（分类）要素如何使用 `[category1, category2, ...]` 有助于预测的价值的学习模型 `tag`.
|
||||
结果指示离散（分类）要素如何使用 `[category1, category2, ...]` 有助于使用学习模型预测`tag`的值。
|
||||
|
||||
## simpleLinearRegression {#simplelinearregression}
|
||||
|
||||
@ -1692,12 +1690,12 @@ simpleLinearRegression(x, y)
|
||||
|
||||
参数:
|
||||
|
||||
- `x` — Column with dependent variable values.
|
||||
- `y` — Column with explanatory variable values.
|
||||
- `x` — x轴。
|
||||
- `y` — y轴。
|
||||
|
||||
返回值:
|
||||
|
||||
常量 `(a, b)` 结果行的 `y = a*x + b`.
|
||||
符合`y = a*x + b`的常量 `(a, b)` 。
|
||||
|
||||
**例**
|
||||
|
||||
@ -1721,9 +1719,9 @@ SELECT arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [3, 4, 5, 6])
|
||||
└───────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## 随机指标线上回归 {#agg_functions-stochasticlinearregression}
|
||||
## stochasticLinearRegression {#agg_functions-stochasticlinearregression}
|
||||
|
||||
该函数实现随机线性回归。 它支持自定义参数的学习率，L2正则化系数，迷你批量大小，并具有更新权重的方法很少 ([亚当](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) （默认使用), [简单SGD](https://en.wikipedia.org/wiki/Stochastic_gradient_descent), [动量](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum), [Nesterov](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)).
|
||||
该函数实现随机线性回归。 它支持自定义参数的学习率、L2正则化系数、微批，并且具有少量更新权重的方法（[Adam](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) （默认）， [simple SGD](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)， [Momentum](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum)， [Nesterov](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)）。
|
||||
|
||||
### 参数 {#agg_functions-stochasticlinearregression-parameters}
|
||||
|
||||
@ -1738,14 +1736,14 @@ stochasticLinearRegression(1.0, 1.0, 10, 'SGD')
|
||||
3. `mini-batch size` 设置元素的数量，这些元素将被计算和求和以执行梯度下降的一个步骤。 纯随机下降使用一个元素，但是具有小批量（约10个元素）使梯度步骤更稳定。 默认值为 `15`.
|
||||
4. `method for updating weights` 他们是: `Adam` （默认情况下), `SGD`, `Momentum`, `Nesterov`. `Momentum` 和 `Nesterov` 需要更多的计算和内存，但是它们恰好在收敛速度和随机梯度方法的稳定性方面是有用的。
|
||||
|
||||
### 用途 {#agg_functions-stochasticlinearregression-usage}
|
||||
### 用法 {#agg_functions-stochasticlinearregression-usage}
|
||||
|
||||
`stochasticLinearRegression` 用于两个步骤：拟合模型和预测新数据。 为了拟合模型并保存其状态以供以后使用，我们使用 `-State` combinator，它基本上保存了状态（模型权重等）。
|
||||
为了预测我们使用函数 [evalMLMethod](../functions/machine-learning-functions.md#machine_learning_methods-evalmlmethod)，这需要一个状态作为参数以及特征来预测。
|
||||
|
||||
<a name="stochasticlinearregression-usage-fitting"></a>
|
||||
|
||||
**1.** 适合
|
||||
**1.** 安装
|
||||
|
||||
可以使用这种查询。
|
||||
|
||||
@ -1807,28 +1805,28 @@ evalMLMethod(model, param1, param2) FROM test_data
|
||||
stochasticLogisticRegression(1.0, 1.0, 10, 'SGD')
|
||||
```
|
||||
|
||||
1. 适合
|
||||
**1.** 安装
|
||||
|
||||
<!-- -->
|
||||
|
||||
See the `Fitting` section in the [stochasticLinearRegression](#stochasticlinearregression-usage-fitting) description.
|
||||
参考stochasticLinearRegression相关文档
|
||||
|
||||
Predicted labels have to be in \[-1, 1\].
|
||||
预测标签的取值范围为[-1, 1]
|
||||
|
||||
1. 预测
|
||||
**2.** 预测
|
||||
|
||||
<!-- -->
|
||||
|
||||
Using saved state we can predict probability of object having label `1`.
|
||||
使用已经保存的state我们可以预测标签为 `1` 的对象的概率。
|
||||
|
||||
``` sql
|
||||
WITH (SELECT state FROM your_model) AS model SELECT
|
||||
evalMLMethod(model, param1, param2) FROM test_data
|
||||
```
|
||||
|
||||
The query will return a column of probabilities. Note that first argument of `evalMLMethod` is `AggregateFunctionState` object, next are columns of features.
|
||||
查询结果返回一个列的概率。注意 `evalMLMethod` 的第一个参数是 `AggregateFunctionState` 对象，接下来的参数是列的特性。
|
||||
|
||||
We can also set a bound of probability, which assigns elements to different labels.
|
||||
我们也可以设置概率的范围， 这样需要给元素指定不同的标签。
|
||||
|
||||
``` sql
|
||||
SELECT ans < 1.1 AND ans > 0.5 FROM
|
||||
@ -1836,14 +1834,14 @@ stochasticLogisticRegression(1.0, 1.0, 10, 'SGD')
|
||||
evalMLMethod(model, param1, param2) AS ans FROM test_data)
|
||||
```
|
||||
|
||||
Then the result will be labels.
|
||||
结果是标签。
|
||||
|
||||
`test_data` is a table like `train_data` but may not contain target value.
|
||||
`test_data` 是一个像 `train_data` 一样的表，但是不包含目标值。
|
||||
|
||||
**另请参阅**
|
||||
|
||||
- [随机指标线上回归](#agg_functions-stochasticlinearregression)
|
||||
- [线性回归和逻辑回归之间的差异。](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression)
|
||||
- [线性回归和逻辑回归之间的差异](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression)
|
||||
|
||||
## groupBitmapAnd {#groupbitmapand}
|
||||
|
||||
|
||||
@ -1,156 +1,162 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_priority: 31
|
||||
toc_title: "\u8BED\u6CD5"
|
||||
toc_title: SQL语法
|
||||
---
|
||||
|
||||
# 语法 {#syntax}
|
||||
|
||||
系统中有两种类型的解析器：完整SQL解析器（递归下降解析器）和数据格式解析器（快速流解析器）。
|
||||
在所有情况下，除了 `INSERT` 查询时，只使用完整的SQL解析器。
|
||||
该 `INSERT` 查询使用两个解析器:
|
||||
# SQL语法 {#syntax}
|
||||
|
||||
CH有2类解析器：完整SQL解析器（递归式解析器），以及数据格式解析器（快速流式解析器）
|
||||
除了 `INSERT` 查询，其它情况下仅使用完整SQL解析器。
|
||||
`INSERT`查询会同时使用2种解析器：
|
||||
``` sql
|
||||
INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def')
|
||||
```
|
||||
|
||||
该 `INSERT INTO t VALUES` 片段由完整的解析器解析，并且数据 `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` 由快速流解析器解析。 您也可以通过使用 [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) 设置。 当 `input_format_values_interpret_expressions = 1`，ClickHouse首先尝试使用fast stream解析器解析值。 如果失败，ClickHouse将尝试对数据使用完整的解析器，将其视为SQL [表达式](#syntax-expressions).
|
||||
含`INSERT INTO t VALUES` 的部分由完整SQL解析器处理，包含数据的部分 `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` 交给快速流式解析器解析。通过设置参数 [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions)，你也可以对数据部分开启完整SQL解析器。当 `input_format_values_interpret_expressions = 1` 时，CH优先采用快速流式解析器来解析数据。如果失败，CH再尝试用完整SQL解析器来处理，就像处理SQL [expression](#syntax-expressions) 一样。
|
||||
|
||||
数据可以有任何格式。 当接收到查询时，服务器计算不超过 [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) RAM中请求的字节（默认为1MB），其余的是流解析。
|
||||
它允许避免与大的问题 `INSERT` 查询。
|
||||
数据可以采用任何格式。当CH接受到请求时，服务端先在内存中计算不超过 [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) 字节的请求数据（默认1 mb），然后剩下部分交给快速流式解析器。
|
||||
|
||||
使用时 `Values` 格式为 `INSERT` 查询，它可能看起来数据被解析相同的表达式 `SELECT` 查询，但事实并非如此。 该 `Values` 格式更为有限。
|
||||
这将避免在处理大型的 `INSERT`语句时出现问题。
|
||||
|
||||
本文的其余部分将介绍完整的解析器。 有关格式解析器的详细信息，请参阅 [格式](../interfaces/formats.md) 科。
|
||||
当 `INSERT` 语句中使用 `Values` 形式时，看起来 数据部分的解析和解析`SELECT` 中的表达式相同，但并不是这样的。 `Values` 形式非常有限。
|
||||
该篇的剩余部分涵盖了完整SQL解析器。关于格式解析的更多信息，参见 [Formats](../interfaces/formats.md) 章节。
|
||||
|
||||
## 空间 {#spaces}
|
||||
## 空字符 {#spaces}
|
||||
|
||||
语法结构之间可能有任意数量的空格符号（包括查询的开始和结束）。 空格符号包括空格、制表符、换行符、CR和换页符。
|
||||
sql语句中（包含sql的起始和结束）可以有任意的空字符，这些空字符类型包括：空格字符，tab制表符，换行符，CR符，换页符等。
|
||||
|
||||
## 评论 {#comments}
|
||||
## 注释 {#comments}
|
||||
|
||||
ClickHouse支持SQL风格和C风格的注释。
|
||||
SQL风格的注释以下开头 `--` 并继续到线的末尾，一个空格后 `--` 可以省略。
|
||||
C型是从 `/*` 到 `*/`并且可以是多行，也不需要空格。
|
||||
CH支持SQL风格或C语言风格的注释：
|
||||
- SQL风格的注释以 `--` 开始，直到行末，`--` 后紧跟的空格可以忽略
|
||||
- C语言风格的注释以 `/*` 开始，以 `*/` 结束，支持多行形式，同样可以省略 `/*` 后的空格
|
||||
|
||||
## 关键词 {#syntax-keywords}
|
||||
## 关键字 {#syntax-keywords}
|
||||
|
||||
当关键字对应于以下关键字时，不区分大小写:
|
||||
以下场景的关键字是大小写不敏感的：
|
||||
- 标准SQL。例如，`SELECT`, `select` 和 `SeLeCt` 都是允许的
|
||||
- 在某些流行的RDBMS中被实现的关键字，例如，`DateTime` 和 `datetime`是一样的
|
||||
|
||||
- SQL标准。 例如, `SELECT`, `select` 和 `SeLeCt` 都是有效的。
|
||||
- 在一些流行的DBMS（MySQL或Postgres）中实现。 例如, `DateTime` 是一样的 `datetime`.
|
||||
|
||||
数据类型名称是否区分大小写可以在 `system.data_type_families` 桌子
|
||||
你可以在系统表 [system.data_type_families](../operations/system-tables.md#system_tables-data_type_families) 中检查某个数据类型的名称是否是大小写敏感型。
|
||||
|
||||
与标准SQL相比，所有其他关键字（包括函数名称）都是 **区分大小写**.
|
||||
和标准SQL相反，所有其它的关键字都是 **大小写敏感的**，包括函数名称。
|
||||
In contrast to standard SQL, all other keywords (including functions names) are **case-sensitive**.
|
||||
|
||||
不保留关键字；它们仅在相应的上下文中被视为保留关键字。 如果您使用 [标识符](#syntax-identifiers) 使用与关键字相同的名称，将它们括在双引号或反引号中。 例如，查询 `SELECT "FROM" FROM table_name` 是有效的，如果表 `table_name` 具有名称的列 `"FROM"`.
|
||||
关键字不是保留的；它们仅在相应的上下文中才会被处理。如果你使用和关键字同名的 [变量名](#syntax-identifiers) ，需要使用双引号或转移符将它们包含起来。例如：如果表 `table_name` 包含列 `"FROM"`，那么 `SELECT "FROM" FROM table_name` 是合法的
|
||||
|
||||
## 标识符 {#syntax-identifiers}
|
||||
## 变量名 {#syntax-identifiers}
|
||||
|
||||
标识符是:
|
||||
变量包括：
|
||||
Identifiers are:
|
||||
|
||||
- 集群、数据库、表、分区和列名称。
|
||||
- 功能。
|
||||
- 数据类型。
|
||||
- [表达式别名](#syntax-expression_aliases).
|
||||
- 集群，数据库，表，分区，列名称
|
||||
- 函数
|
||||
- 数据类型
|
||||
- 表达式别名
|
||||
|
||||
标识符可以是引号或非引号。 后者是优选的。
|
||||
变量名可以使用反引号包含起来
|
||||
|
||||
非引号标识符必须与正则表达式匹配 `^[a-zA-Z_][0-9a-zA-Z_]*$` 并且不能等于 [关键词](#syntax-keywords). 例: `x, _1, X_y__Z123_.`
|
||||
没有使用反引号包含的变量名，必须匹配正则表达式 `^[a-zA-Z_][0-9a-zA-Z_]*$`，并且不能和 [关键字]相同
|
||||
|
||||
如果要使用与关键字相同的标识符，或者要在标识符中使用其他符号，请使用双引号或反引号对其进行引用，例如, `"id"`, `` `id` ``.
|
||||
如果想使用和关键字同名的变量名称，或者在变量名称中包含其它符号，你需要通过双引号或转义符号，例如： `"id"`, `` `id` ``
|
||||
|
||||
## 文字数 {#literals}
|
||||
## 字符 {#literals}
|
||||
|
||||
有数字，字符串，复合和 `NULL` 文字。
|
||||
CH包含数字，字母，括号，NULL值等字符
|
||||
|
||||
### 数字 {#numeric}
|
||||
|
||||
数值文字尝试进行分析:
|
||||
数字类型字符会被做如下解析：
|
||||
- 首先，当做64位的有符号整数，使用该函数 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul)
|
||||
- 如果失败，解析成64位无符号整数，同样使用函数 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul)
|
||||
|
||||
- 首先，作为一个64位有符号的数字，使用 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) 功能。
|
||||
- 如果不成功，作为64位无符号数，使用 [strtoll](https://en.cppreference.com/w/cpp/string/byte/strtol) 功能。
|
||||
- 如果不成功，作为一个浮点数使用 [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) 功能。
|
||||
- 否则，将返回错误。
|
||||
- 如果还失败了，试图解析成浮点型数值，使用函数 [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof)
|
||||
Numeric literal tries to be parsed:
|
||||
|
||||
文本值具有该值适合的最小类型。
|
||||
例如，1被解析为 `UInt8`，但256被解析为 `UInt16`. 有关详细信息，请参阅 [数据类型](../sql-reference/data-types/index.md).
|
||||
- 最后，以上情形都不符合时，返回异常
|
||||
|
||||
例: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`.
|
||||
|
||||
### 字符串 {#syntax-string-literal}
|
||||
数字类型的值类型为能容纳该值的最小数据类型。
|
||||
例如：1 解析成 `UInt8`型，256 则解析成 `UInt16`。更多信息，参见 [数据类型](../sql-reference/data-types/index.md)
|
||||
|
||||
仅支持单引号中的字符串文字。 封闭的字符可以反斜杠转义。 以下转义序列具有相应的特殊值: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. 在所有其他情况下，转义序列的格式为 `\c`，哪里 `c` 是任何字符，被转换为 `c`. 这意味着你可以使用序列 `\'`和`\\`. 该值将具有 [字符串](../sql-reference/data-types/string.md) 类型。
|
||||
例如: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`.
|
||||
|
||||
在字符串文字中，你至少需要转义 `'` 和 `\`. 单引号可以用单引号，文字转义 `'It\'s'` 和 `'It''s'` 是平等的。
|
||||
### 字母 {#syntax-string-literal}
|
||||
CH只支持用单引号包含的字母。特殊字符可通过反斜杠进行转义。下列转义字符都有相应的实际值： `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`。其它情况下，以 `\c`形式出现的转义字符，当`c`表示任意字符时，转义字符会转换成`c`。这意味着你可以使用 `\'`和`\\`。该值将拥有[String](../sql-reference/data-types/string.md)类型。
|
||||
|
||||
### 化合物 {#compound}
|
||||
|
||||
数组使用方括号构造 `[1, 2, 3]`. Nuples用圆括号构造 `(1, 'Hello, world!', 2)`.
|
||||
从技术上讲，这些不是文字，而是分别具有数组创建运算符和元组创建运算符的表达式。
|
||||
数组必须至少包含一个项目，元组必须至少包含两个项目。
|
||||
有一个单独的情况下，当元组出现在 `IN` a条款 `SELECT` 查询。 查询结果可以包含元组，但元组不能保存到数据库（除了具有以下内容的表 [记忆](../engines/table-engines/special/memory.md) 发动机）。
|
||||
在字符串中，你至少需要对 `'` 和 `\` 进行转义。单引号可以使用单引号转义，例如 `'It\'s'` 和 `'It''s'` 是相同的。
|
||||
|
||||
### NULL {#null-literal}
|
||||
### 括号 {#compound}
|
||||
数组都是使用方括号进行构造 `[1, 2, 3]`，元组则使用圆括号 `(1, 'Hello, world!', 2)`
|
||||
|
||||
指示该值丢失。
|
||||
从技术上来讲，这些都不是字符串，而是包含创建数组和元组运算符的表达式。
|
||||
|
||||
为了存储 `NULL` 在表字段中，它必须是 [可为空](../sql-reference/data-types/nullable.md) 类型。
|
||||
创建一个数组必须至少包含一个元素，创建一个元组至少包含2个元素
|
||||
|
||||
根据数据格式（输入或输出), `NULL` 可能有不同的表示。 有关详细信息，请参阅以下文档 [数据格式](../interfaces/formats.md#formats).
|
||||
当元组出现在 `SELECT` 查询的 `IN` 部分时，是一种例外情形。查询结果可以包含元组，但是元组类型不能保存到数据库中（除非表采用 [内存表](../engines/table-engines/special/memory.md)引擎）
|
||||
|
||||
处理有许多细微差别 `NULL`. 例如，如果比较操作的至少一个参数是 `NULL`，此操作的结果也是 `NULL`. 对于乘法，加法和其他操作也是如此。 有关详细信息，请阅读每个操作的文档。
|
||||
|
||||
在查询中，您可以检查 `NULL` 使用 [IS NULL](operators/index.md#operator-is-null) 和 [IS NOT NULL](operators/index.md) 运算符及相关功能 `isNull` 和 `isNotNull`.
|
||||
### NULL值 {#null-literal}
|
||||
|
||||
## 功能 {#functions}
|
||||
代表不存在的值
|
||||
|
||||
函数调用像一个标识符一样写入，并在圆括号中包含一个参数列表（可能是空的）。 与标准SQL相比，括号是必需的，即使是空的参数列表。 示例: `now()`.
|
||||
有常规函数和聚合函数（请参阅部分 “Aggregate functions”). 某些聚合函数可以包含括号中的两个参数列表。 示例: `quantile (0.9) (x)`. 这些聚合函数被调用 “parametric” 函数，并在第一个列表中的参数被调用 “parameters”. 不带参数的聚合函数的语法与常规函数的语法相同。
|
||||
为了能在表字段中存储NULL值，该字段必须声明为 [空值](../sql-reference/data-types/nullable.md) 类型
|
||||
根据数据的格式（输入或输出），NULL值有不同的表现形式。更多信息参见文档 [数据格式](../interfaces/formats.md#formats)
|
||||
|
||||
## 运营商 {#operators}
|
||||
在处理 `NULL`时存在很多细微差别。例如，比较运算的至少一个参数为 `NULL` ，该结果也是 `NULL` 。与之类似的还有乘法运算, 加法运算,以及其它运算。更多信息，请参阅每种运算的文档部分。
|
||||
|
||||
在查询解析过程中，运算符会转换为相应的函数，同时考虑它们的优先级和关联性。
|
||||
例如，表达式 `1 + 2 * 3 + 4` 转化为 `plus(plus(1, multiply(2, 3)), 4)`.
|
||||
在语句中，可以通过 [是否为NULL](operators/index.md#operator-is-null) 以及 [是否不为NULL](operators/index.md) 运算符，以及 `isNull` 、 `isNotNull` 函数来检查 `NULL` 值
|
||||
|
||||
## 数据类型和数据库表引擎 {#data_types-and-database-table-engines}
|
||||
## 函数 {#functions}
|
||||
函数调用的写法，类似于变量并带有被圆括号包含的参数列表（可能为空）。与标准SQL不同，圆括号是必须的，不管参数列表是否为空。例如： `now()`。
|
||||
|
||||
数据类型和表引擎 `CREATE` 查询的编写方式与标识符或函数相同。 换句话说，它们可能包含也可能不包含括号中的参数列表。 有关详细信息，请参阅部分 “Data types,” “Table engines,” 和 “CREATE”.
|
||||
函数分为常规函数和聚合函数（参见“Aggregate functions”一章）。有些聚合函数包含2个参数列表，第一个参数列表中的参数被称为“parameters”。不包含“parameters”的聚合函数语法和常规函数是一样的。
|
||||
|
||||
|
||||
## 运算符 {#operators}
|
||||
|
||||
在查询解析阶段，运算符会被转换成对应的函数，使用时请注意它们的优先级。例如：
|
||||
表达式 `1 + 2 * 3 + 4` 会被解析成 `plus(plus(1, multiply(2, 3)), 4)`.
|
||||
|
||||
|
||||
## 数据类型及数据库/表引擎 {#data_types-and-database-table-engines}
|
||||
|
||||
`CREATE` 语句中的数据类型和表引擎写法与变量或函数类似。
|
||||
换句话说，它们可以用括号包含参数列表。更多信息，参见“数据类型,” “数据表引擎” 和 “CREATE语句”等章节
|
||||
|
||||
## 表达式别名 {#syntax-expression_aliases}
|
||||
|
||||
别名是查询中表达式的用户定义名称。
|
||||
别名是用户对表达式的自定义名称
|
||||
|
||||
``` sql
|
||||
expr AS alias
|
||||
```
|
||||
|
||||
- `AS` — The keyword for defining aliases. You can define the alias for a table name or a column name in a `SELECT` 子句不使用 `AS` 关键字。
|
||||
- `AS` — 用于定义别名的关键字。可以对表或select语句中的列定义别名(`AS` 可以省略）
|
||||
例如, `SELECT table_name_alias.column_name FROM table_name table_name_alias`.
|
||||
|
||||
For example, `SELECT table_name_alias.column_name FROM table_name table_name_alias`.
|
||||
在 [CAST函数](sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) 中，`AS`有其它含义。请参见该函数的说明部分。
|
||||
|
||||
In the [CAST](sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) function, the `AS` keyword has another meaning. See the description of the function.
|
||||
|
||||
- `expr` — Any expression supported by ClickHouse.
|
||||
- `expr` — 任意CH支持的表达式.
|
||||
|
||||
For example, `SELECT column_name * 2 AS double FROM some_table`.
|
||||
例如, `SELECT column_name * 2 AS double FROM some_table`.
|
||||
|
||||
- `alias` — Name for `expr`. 别名应符合 [标识符](#syntax-identifiers) 语法
|
||||
- `alias` — `expr` 的名称。别名必须符合 [变量名]](#syntax-identifiers) 语法.
|
||||
|
||||
For example, `SELECT "table t".column_name FROM table_name AS "table t"`.
|
||||
例如, `SELECT "table t".column_name FROM table_name AS "table t"`.
|
||||
|
||||
### 使用注意事项 {#notes-on-usage}
|
||||
### 用法注意 {#notes-on-usage}
|
||||
|
||||
别名对于查询或子查询是全局的，您可以在查询的任何部分中为任何表达式定义别名。 例如, `SELECT (1 AS n) + 2, n`.
|
||||
别名在当前查询或子查询中是全局可见的，你可以在查询语句的任何位置对表达式定义别名
|
||||
|
||||
别名在子查询和子查询之间不可见。 例如，在执行查询时 `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` ClickHouse生成异常 `Unknown identifier: num`.
|
||||
别名在当前查询的子查询及不同子查询中是不可见的。例如，执行如下查询SQL: `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` ,CH会提示异常 `Unknown identifier: num`.
|
||||
|
||||
如果为结果列定义了别名 `SELECT` 子查询的子句，这些列在外部查询中可见。 例如, `SELECT n + m FROM (SELECT 1 AS n, 2 AS m)`.
|
||||
|
||||
小心使用与列或表名相同的别名。 让我们考虑以下示例:
|
||||
如果给select子查询语句的结果列定义其别名，那么在外层可以使用该别名。例如, `SELECT n + m FROM (SELECT 1 AS n, 2 AS m)`.
|
||||
|
||||
注意列的别名和表的别名相同时的情形，考虑如下示例：
|
||||
``` sql
|
||||
CREATE TABLE t
|
||||
(
|
||||
@ -172,16 +178,18 @@ Received exception from server (version 18.14.17):
|
||||
Code: 184. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: Aggregate function sum(b) is found inside another aggregate function in query.
|
||||
```
|
||||
|
||||
在这个例子中，我们声明表 `t` 带柱 `b`. 然后，在选择数据时，我们定义了 `sum(b) AS b` 别名 由于别名是全局的，ClickHouse替换了文字 `b` 在表达式中 `argMax(a, b)` 用表达式 `sum(b)`. 这种替换导致异常。
|
||||
在这个示例中，先声明了表 `t` 以及列 `b`。然后，在查询数据时，又定义了别名 `sum(b) AS b`。由于别名是全局的，CH使用表达式 `sum(b)` 来替换表达式 `argMax(a, b)` 中的变量 `b`。这种替换导致出现异常。
|
||||
|
||||
## 星号 {#asterisk}
|
||||
|
||||
在一个 `SELECT` 查询中，星号可以替换表达式。 有关详细信息，请参阅部分 “SELECT”.
|
||||
select查询中，星号可以代替表达式使用。详情请参见“select”部分
|
||||
|
||||
|
||||
## 表达式 {#syntax-expressions}
|
||||
|
||||
表达式是函数、标识符、文字、运算符的应用程序、括号中的表达式、子查询或星号。 它还可以包含别名。
|
||||
表达式列表是一个或多个用逗号分隔的表达式。
|
||||
函数和运算符，反过来，可以有表达式作为参数。
|
||||
|
||||
[原始文章](https://clickhouse.tech/docs/en/sql_reference/syntax/) <!--hide-->
|
||||
An expression is a function, identifier, literal, application of an operator, expression in brackets, subquery, or asterisk. It can also contain an alias.
|
||||
A list of expressions is one or more expressions separated by commas.
|
||||
Functions and operators, in turn, can have expressions as arguments.
|
||||
|
||||
[原始文档](https://clickhouse.tech/docs/en/sql_reference/syntax/) <!--hide-->
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
#include <Common/ConcurrentBoundedQueue.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <Common/clearPasswordFromCommandLine.h>
|
||||
#include <Core/Types.h>
|
||||
#include <IO/ReadBufferFromFileDescriptor.h>
|
||||
#include <IO/WriteBufferFromFileDescriptor.h>
|
||||
@ -571,6 +572,8 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
|
||||
boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options);
|
||||
boost::program_options::notify(options);
|
||||
|
||||
clearPasswordFromCommandLine(argc, argv);
|
||||
|
||||
if (options.count("help"))
|
||||
{
|
||||
std::cout << "Usage: " << argv[0] << " [options] < queries.txt\n";
|
||||
|
||||
@ -38,6 +38,7 @@
|
||||
#include <Common/Throttler.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/clearPasswordFromCommandLine.h>
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Core/QueryProcessingStage.h>
|
||||
@ -2006,6 +2007,7 @@ public:
|
||||
|
||||
argsToConfig(common_arguments, config(), 100);
|
||||
|
||||
clearPasswordFromCommandLine(argc, argv);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -25,7 +25,7 @@ void ClusterCopier::init()
|
||||
|
||||
task_description_watch_callback = [this] (const Coordination::WatchResponse & response)
|
||||
{
|
||||
if (response.error != Coordination::ZOK)
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
return;
|
||||
UInt64 version = ++task_description_version;
|
||||
LOG_DEBUG(log, "Task description should be updated, local version {}", version);
|
||||
@ -206,11 +206,11 @@ void ClusterCopier::uploadTaskDescription(const std::string & task_path, const s
|
||||
|
||||
zookeeper->createAncestors(local_task_description_path);
|
||||
auto code = zookeeper->tryCreate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent);
|
||||
if (code && force)
|
||||
if (code != Coordination::Error::ZOK && force)
|
||||
zookeeper->createOrUpdate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent);
|
||||
|
||||
LOG_DEBUG(log, "Task description {} uploaded to {} with result {} ({})",
|
||||
((code && !force) ? "not " : ""), local_task_description_path, code, zookeeper->error2string(code));
|
||||
((code != Coordination::Error::ZOK && !force) ? "not " : ""), local_task_description_path, code, Coordination::errorMessage(code));
|
||||
}
|
||||
|
||||
void ClusterCopier::reloadTaskDescription()
|
||||
@ -220,10 +220,10 @@ void ClusterCopier::reloadTaskDescription()
|
||||
|
||||
String task_config_str;
|
||||
Coordination::Stat stat{};
|
||||
int code;
|
||||
Coordination::Error code;
|
||||
|
||||
zookeeper->tryGetWatch(task_description_path, task_config_str, &stat, task_description_watch_callback, &code);
|
||||
if (code)
|
||||
if (code != Coordination::Error::ZOK)
|
||||
throw Exception("Can't get description node " + task_description_path, ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
LOG_DEBUG(log, "Loading description, zxid={}", task_description_current_stat.czxid);
|
||||
@ -376,10 +376,10 @@ zkutil::EphemeralNodeHolder::Ptr ClusterCopier::createTaskWorkerNodeAndWaitIfNee
|
||||
Coordination::Responses responses;
|
||||
auto code = zookeeper->tryMulti(ops, responses);
|
||||
|
||||
if (code == Coordination::ZOK || code == Coordination::ZNODEEXISTS)
|
||||
if (code == Coordination::Error::ZOK || code == Coordination::Error::ZNODEEXISTS)
|
||||
return std::make_shared<zkutil::EphemeralNodeHolder>(current_worker_path, *zookeeper, false, false, description);
|
||||
|
||||
if (code == Coordination::ZBADVERSION)
|
||||
if (code == Coordination::Error::ZBADVERSION)
|
||||
{
|
||||
++num_bad_version_errors;
|
||||
|
||||
@ -545,7 +545,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t
|
||||
}
|
||||
catch (const Coordination::Exception & e)
|
||||
{
|
||||
if (e.code == Coordination::ZNODEEXISTS)
|
||||
if (e.code == Coordination::Error::ZNODEEXISTS)
|
||||
{
|
||||
LOG_DEBUG(log, "Someone is already moving pieces {}", current_partition_attach_is_active);
|
||||
return TaskStatus::Active;
|
||||
@ -745,7 +745,7 @@ bool ClusterCopier::tryDropPartitionPiece(
|
||||
}
|
||||
catch (const Coordination::Exception & e)
|
||||
{
|
||||
if (e.code == Coordination::ZNODEEXISTS)
|
||||
if (e.code == Coordination::Error::ZNODEEXISTS)
|
||||
{
|
||||
LOG_DEBUG(log, "Partition {} piece {} is cleaning now by somebody, sleep", task_partition.name, toString(current_piece_number));
|
||||
std::this_thread::sleep_for(default_sleep_time);
|
||||
@ -778,7 +778,7 @@ bool ClusterCopier::tryDropPartitionPiece(
|
||||
}
|
||||
catch (const Coordination::Exception & e)
|
||||
{
|
||||
if (e.code == Coordination::ZNODEEXISTS)
|
||||
if (e.code == Coordination::Error::ZNODEEXISTS)
|
||||
{
|
||||
LOG_DEBUG(log, "Partition {} is being filled now by somebody, sleep", task_partition.name);
|
||||
return false;
|
||||
@ -795,7 +795,7 @@ bool ClusterCopier::tryDropPartitionPiece(
|
||||
/// Remove all status nodes
|
||||
{
|
||||
Strings children;
|
||||
if (zookeeper->tryGetChildren(current_shards_path, children) == Coordination::ZOK)
|
||||
if (zookeeper->tryGetChildren(current_shards_path, children) == Coordination::Error::ZOK)
|
||||
for (const auto & child : children)
|
||||
{
|
||||
zookeeper->removeRecursive(current_shards_path + "/" + child);
|
||||
@ -845,7 +845,7 @@ bool ClusterCopier::tryDropPartitionPiece(
|
||||
}
|
||||
|
||||
LOG_INFO(log, "Partition {} piece {} was dropped on cluster {}", task_partition.name, toString(current_piece_number), task_table.cluster_push_name);
|
||||
if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::ZNODEEXISTS)
|
||||
if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::Error::ZNODEEXISTS)
|
||||
zookeeper->set(current_shards_path, host_id);
|
||||
}
|
||||
|
||||
@ -1233,7 +1233,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
|
||||
}
|
||||
catch (const Coordination::Exception & e)
|
||||
{
|
||||
if (e.code == Coordination::ZNODEEXISTS)
|
||||
if (e.code == Coordination::Error::ZNODEEXISTS)
|
||||
{
|
||||
LOG_DEBUG(log, "Someone is already processing {}", current_task_piece_is_active_path);
|
||||
return TaskStatus::Active;
|
||||
@ -1271,9 +1271,9 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
|
||||
{
|
||||
String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id);
|
||||
auto res = zookeeper->tryCreate(current_task_piece_status_path, state_finished, zkutil::CreateMode::Persistent);
|
||||
if (res == Coordination::ZNODEEXISTS)
|
||||
if (res == Coordination::Error::ZNODEEXISTS)
|
||||
LOG_DEBUG(log, "Partition {} piece {} is absent on current replica of a shard. But other replicas have already marked it as done.", task_partition.name, current_piece_number);
|
||||
if (res == Coordination::ZOK)
|
||||
if (res == Coordination::Error::ZOK)
|
||||
LOG_DEBUG(log, "Partition {} piece {} is absent on current replica of a shard. Will mark it as done. Other replicas will do the same.", task_partition.name, current_piece_number);
|
||||
return TaskStatus::Finished;
|
||||
}
|
||||
@ -1429,7 +1429,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
|
||||
{
|
||||
Coordination::ExistsResponse status = future_is_dirty_checker.get();
|
||||
|
||||
if (status.error != Coordination::ZNONODE)
|
||||
if (status.error != Coordination::Error::ZNONODE)
|
||||
{
|
||||
LogicalClock dirt_discovery_epoch (status.stat.mzxid);
|
||||
if (dirt_discovery_epoch == clean_state_clock.discovery_zxid)
|
||||
|
||||
@ -178,7 +178,7 @@ public:
|
||||
[stale = stale] (const Coordination::WatchResponse & rsp)
|
||||
{
|
||||
auto logger = &Poco::Logger::get("ClusterCopier");
|
||||
if (rsp.error == Coordination::ZOK)
|
||||
if (rsp.error == Coordination::Error::ZOK)
|
||||
{
|
||||
switch (rsp.type)
|
||||
{
|
||||
|
||||
7
programs/server/users.d/access_management.xml
Normal file
7
programs/server/users.d/access_management.xml
Normal file
@ -0,0 +1,7 @@
|
||||
<yandex>
|
||||
<users>
|
||||
<default>
|
||||
<access_management>1</access_management>
|
||||
</default>
|
||||
</users>
|
||||
</yandex>
|
||||
@ -67,6 +67,12 @@ public:
|
||||
{
|
||||
data(place).count = new_count;
|
||||
}
|
||||
|
||||
/// The function returns non-Nullable type even when wrapped with Null combinator.
|
||||
bool returnDefaultWhenOnlyNull() const override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -63,14 +63,15 @@ AggregateFunctionPtr AggregateFunctionFactory::get(
|
||||
{
|
||||
auto type_without_low_cardinality = convertLowCardinalityTypesToNested(argument_types);
|
||||
|
||||
/// If one of types is Nullable, we apply aggregate function combinator "Null".
|
||||
/// If one of the types is Nullable, we apply aggregate function combinator "Null".
|
||||
|
||||
if (std::any_of(type_without_low_cardinality.begin(), type_without_low_cardinality.end(),
|
||||
[](const auto & type) { return type->isNullable(); }))
|
||||
{
|
||||
AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix("Null");
|
||||
if (!combinator)
|
||||
throw Exception("Logical error: cannot find aggregate function combinator to apply a function to Nullable arguments.", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Logical error: cannot find aggregate function combinator to apply a function to Nullable arguments.",
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
DataTypes nested_types = combinator->transformArguments(type_without_low_cardinality);
|
||||
Array nested_parameters = combinator->transformParameters(parameters);
|
||||
@ -132,9 +133,10 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl(
|
||||
|
||||
auto hints = this->getHints(name);
|
||||
if (!hints.empty())
|
||||
throw Exception("Unknown aggregate function " + name + ". Maybe you meant: " + toString(hints), ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION);
|
||||
throw Exception(fmt::format("Unknown aggregate function {}. Maybe you meant: {}", name, toString(hints)),
|
||||
ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION);
|
||||
else
|
||||
throw Exception("Unknown aggregate function " + name, ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION);
|
||||
throw Exception(fmt::format("Unknown aggregate function {}", name), ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -33,6 +33,11 @@ public:
|
||||
AggregateFunctionPtr transformAggregateFunction(
|
||||
const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params) const override
|
||||
{
|
||||
/// Special case for 'count' function. It could be called with Nullable arguments
|
||||
/// - that means - count number of calls, when all arguments are not NULL.
|
||||
if (nested_function && nested_function->getName() == "count")
|
||||
return std::make_shared<AggregateFunctionCountNotNullUnary>(arguments[0], params);
|
||||
|
||||
bool has_nullable_types = false;
|
||||
bool has_null_types = false;
|
||||
for (const auto & arg_type : arguments)
|
||||
@ -49,35 +54,47 @@ public:
|
||||
}
|
||||
|
||||
if (!has_nullable_types)
|
||||
throw Exception("Aggregate function combinator 'Null' requires at least one argument to be Nullable", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
if (nested_function)
|
||||
if (auto adapter = nested_function->getOwnNullAdapter(nested_function, arguments, params))
|
||||
return adapter;
|
||||
|
||||
/// Special case for 'count' function. It could be called with Nullable arguments
|
||||
/// - that means - count number of calls, when all arguments are not NULL.
|
||||
if (nested_function && nested_function->getName() == "count")
|
||||
return std::make_shared<AggregateFunctionCountNotNullUnary>(arguments[0], params);
|
||||
throw Exception("Aggregate function combinator 'Null' requires at least one argument to be Nullable",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
if (has_null_types)
|
||||
return std::make_shared<AggregateFunctionNothing>(arguments, params);
|
||||
|
||||
bool return_type_is_nullable = nested_function->getReturnType()->canBeInsideNullable();
|
||||
assert(nested_function);
|
||||
|
||||
if (auto adapter = nested_function->getOwnNullAdapter(nested_function, arguments, params))
|
||||
return adapter;
|
||||
|
||||
bool return_type_is_nullable = !nested_function->returnDefaultWhenOnlyNull() && nested_function->getReturnType()->canBeInsideNullable();
|
||||
bool serialize_flag = return_type_is_nullable || nested_function->returnDefaultWhenOnlyNull();
|
||||
|
||||
if (arguments.size() == 1)
|
||||
{
|
||||
if (return_type_is_nullable)
|
||||
return std::make_shared<AggregateFunctionNullUnary<true>>(nested_function, arguments, params);
|
||||
{
|
||||
return std::make_shared<AggregateFunctionNullUnary<true, true>>(nested_function, arguments, params);
|
||||
}
|
||||
else
|
||||
return std::make_shared<AggregateFunctionNullUnary<false>>(nested_function, arguments, params);
|
||||
{
|
||||
if (serialize_flag)
|
||||
return std::make_shared<AggregateFunctionNullUnary<false, true>>(nested_function, arguments, params);
|
||||
else
|
||||
return std::make_shared<AggregateFunctionNullUnary<false, false>>(nested_function, arguments, params);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (return_type_is_nullable)
|
||||
return std::make_shared<AggregateFunctionNullVariadic<true, true>>(nested_function, arguments, params);
|
||||
{
|
||||
return std::make_shared<AggregateFunctionNullVariadic<true, true, true>>(nested_function, arguments, params);
|
||||
}
|
||||
else
|
||||
return std::make_shared<AggregateFunctionNullVariadic<false, true>>(nested_function, arguments, params);
|
||||
{
|
||||
if (serialize_flag)
|
||||
return std::make_shared<AggregateFunctionNullVariadic<false, true, true>>(nested_function, arguments, params);
|
||||
else
|
||||
return std::make_shared<AggregateFunctionNullVariadic<false, true, false>>(nested_function, arguments, params);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@ -28,7 +28,10 @@ namespace ErrorCodes
|
||||
/// If all rows had NULL, the behaviour is determined by "result_is_nullable" template parameter.
|
||||
/// true - return NULL; false - return value from empty aggregation state of nested function.
|
||||
|
||||
template <bool result_is_nullable, typename Derived>
|
||||
/// When serialize_flag is set to true, the flag about presense of values is serialized
|
||||
/// regardless to the "result_is_nullable" even if it's unneeded - for protocol compatibility.
|
||||
|
||||
template <bool result_is_nullable, bool serialize_flag, typename Derived>
|
||||
class AggregateFunctionNullBase : public IAggregateFunctionHelper<Derived>
|
||||
{
|
||||
protected:
|
||||
@ -129,7 +132,7 @@ public:
|
||||
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
||||
{
|
||||
bool flag = getFlag(place);
|
||||
if constexpr (result_is_nullable)
|
||||
if constexpr (serialize_flag)
|
||||
writeBinary(flag, buf);
|
||||
if (flag)
|
||||
nested_function->serialize(nestedPlace(place), buf);
|
||||
@ -138,7 +141,7 @@ public:
|
||||
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena * arena) const override
|
||||
{
|
||||
bool flag = 1;
|
||||
if constexpr (result_is_nullable)
|
||||
if constexpr (serialize_flag)
|
||||
readBinary(flag, buf);
|
||||
if (flag)
|
||||
{
|
||||
@ -183,12 +186,15 @@ public:
|
||||
/** There are two cases: for single argument and variadic.
|
||||
* Code for single argument is much more efficient.
|
||||
*/
|
||||
template <bool result_is_nullable>
|
||||
class AggregateFunctionNullUnary final : public AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullUnary<result_is_nullable>>
|
||||
template <bool result_is_nullable, bool serialize_flag>
|
||||
class AggregateFunctionNullUnary final
|
||||
: public AggregateFunctionNullBase<result_is_nullable, serialize_flag,
|
||||
AggregateFunctionNullUnary<result_is_nullable, serialize_flag>>
|
||||
{
|
||||
public:
|
||||
AggregateFunctionNullUnary(AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params)
|
||||
: AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullUnary<result_is_nullable>>(std::move(nested_function_), arguments, params)
|
||||
: AggregateFunctionNullBase<result_is_nullable, serialize_flag,
|
||||
AggregateFunctionNullUnary<result_is_nullable, serialize_flag>>(std::move(nested_function_), arguments, params)
|
||||
{
|
||||
}
|
||||
|
||||
@ -218,12 +224,15 @@ public:
|
||||
};
|
||||
|
||||
|
||||
template <bool result_is_nullable, bool null_is_skipped>
|
||||
class AggregateFunctionNullVariadic final : public AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullVariadic<result_is_nullable, null_is_skipped>>
|
||||
template <bool result_is_nullable, bool serialize_flag, bool null_is_skipped>
|
||||
class AggregateFunctionNullVariadic final
|
||||
: public AggregateFunctionNullBase<result_is_nullable, serialize_flag,
|
||||
AggregateFunctionNullVariadic<result_is_nullable, serialize_flag, null_is_skipped>>
|
||||
{
|
||||
public:
|
||||
AggregateFunctionNullVariadic(AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params)
|
||||
: AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullVariadic<result_is_nullable, null_is_skipped>>(std::move(nested_function_), arguments, params),
|
||||
: AggregateFunctionNullBase<result_is_nullable, serialize_flag,
|
||||
AggregateFunctionNullVariadic<result_is_nullable, serialize_flag, null_is_skipped>>(std::move(nested_function_), arguments, params),
|
||||
number_of_arguments(arguments.size())
|
||||
{
|
||||
if (number_of_arguments == 1)
|
||||
@ -263,11 +272,6 @@ public:
|
||||
this->nested_function->add(this->nestedPlace(place), nested_columns, row_num, arena);
|
||||
}
|
||||
|
||||
bool allocatesMemoryInArena() const override
|
||||
{
|
||||
return this->nested_function->allocatesMemoryInArena();
|
||||
}
|
||||
|
||||
private:
|
||||
enum { MAX_ARGS = 8 };
|
||||
size_t number_of_arguments = 0;
|
||||
|
||||
@ -244,6 +244,12 @@ public:
|
||||
{
|
||||
assert_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
|
||||
}
|
||||
|
||||
/// The function returns non-Nullable type even when wrapped with Null combinator.
|
||||
bool returnDefaultWhenOnlyNull() const override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -298,6 +304,12 @@ public:
|
||||
{
|
||||
assert_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
|
||||
}
|
||||
|
||||
/// The function returns non-Nullable type even when wrapped with Null combinator.
|
||||
bool returnDefaultWhenOnlyNull() const override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -240,9 +240,10 @@ public:
|
||||
return std::make_shared<DataTypeUInt8>();
|
||||
}
|
||||
|
||||
AggregateFunctionPtr getOwnNullAdapter(const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params) const override
|
||||
AggregateFunctionPtr getOwnNullAdapter(
|
||||
const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params) const override
|
||||
{
|
||||
return std::make_shared<AggregateFunctionNullVariadic<false, false>>(nested_function, arguments, params);
|
||||
return std::make_shared<AggregateFunctionNullVariadic<false, false, false>>(nested_function, arguments, params);
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override
|
||||
|
||||
@ -171,6 +171,12 @@ public:
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/** When the function is wrapped with Null combinator,
|
||||
* should we return Nullable type with NULL when no values were aggregated
|
||||
* or we should return non-Nullable type with default value (example: count, countDistinct).
|
||||
*/
|
||||
virtual bool returnDefaultWhenOnlyNull() const { return false; }
|
||||
|
||||
const DataTypes & getArgumentTypes() const { return argument_types; }
|
||||
const Array & getParameters() const { return parameters; }
|
||||
|
||||
|
||||
@ -4,10 +4,10 @@
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <boost/noncopyable.hpp>
|
||||
#if __has_include(<sanitizer/asan_interface.h>)
|
||||
#include <Core/Defines.h>
|
||||
#if __has_include(<sanitizer/asan_interface.h>) && defined(ADDRESS_SANITIZER)
|
||||
# include <sanitizer/asan_interface.h>
|
||||
#endif
|
||||
#include <Core/Defines.h>
|
||||
#include <Common/memcpySmall.h>
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/Allocator.h>
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#if __has_include(<sanitizer/asan_interface.h>)
|
||||
#include <Core/Defines.h>
|
||||
#if __has_include(<sanitizer/asan_interface.h>) && defined(ADDRESS_SANITIZER)
|
||||
# include <sanitizer/asan_interface.h>
|
||||
#endif
|
||||
#include <Core/Defines.h>
|
||||
#include <Common/Arena.h>
|
||||
#include <Common/BitHelpers.h>
|
||||
|
||||
|
||||
@ -54,6 +54,18 @@ Elf::Elf(const std::string & path)
|
||||
throw Exception("The ELF is truncated (section names string table points after end of file)", ErrorCodes::CANNOT_PARSE_ELF);
|
||||
|
||||
section_names = reinterpret_cast<const char *>(mapped + section_names_offset);
|
||||
|
||||
/// Get program headers
|
||||
|
||||
ElfOff program_header_offset = header->e_phoff;
|
||||
uint16_t program_header_num_entries = header->e_phnum;
|
||||
|
||||
if (!program_header_offset
|
||||
|| !program_header_num_entries
|
||||
|| program_header_offset + program_header_num_entries * sizeof(ElfPhdr) > elf_size)
|
||||
throw Exception("The ELF is truncated (program header points after end of file)", ErrorCodes::CANNOT_PARSE_ELF);
|
||||
|
||||
program_headers = reinterpret_cast<const ElfPhdr *>(mapped + program_header_offset);
|
||||
}
|
||||
|
||||
|
||||
@ -104,6 +116,40 @@ std::optional<Elf::Section> Elf::findSectionByName(const char * name) const
|
||||
}
|
||||
|
||||
|
||||
String Elf::getBuildID() const
|
||||
{
|
||||
for (size_t idx = 0; idx < header->e_phnum; ++idx)
|
||||
{
|
||||
const ElfPhdr & phdr = program_headers[idx];
|
||||
|
||||
if (phdr.p_type == PT_NOTE)
|
||||
return getBuildID(mapped + phdr.p_offset, phdr.p_filesz);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
String Elf::getBuildID(const char * nhdr_pos, size_t size)
|
||||
{
|
||||
const char * nhdr_end = nhdr_pos + size;
|
||||
|
||||
while (nhdr_pos < nhdr_end)
|
||||
{
|
||||
const ElfNhdr & nhdr = *reinterpret_cast<const ElfNhdr *>(nhdr_pos);
|
||||
|
||||
nhdr_pos += sizeof(ElfNhdr) + nhdr.n_namesz;
|
||||
if (nhdr.n_type == NT_GNU_BUILD_ID)
|
||||
{
|
||||
const char * build_id = nhdr_pos;
|
||||
return {build_id, nhdr.n_descsz};
|
||||
}
|
||||
nhdr_pos += nhdr.n_descsz;
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
const char * Elf::Section::name() const
|
||||
{
|
||||
if (!elf.section_names)
|
||||
|
||||
@ -17,6 +17,7 @@ using ElfEhdr = ElfW(Ehdr);
|
||||
using ElfOff = ElfW(Off);
|
||||
using ElfPhdr = ElfW(Phdr);
|
||||
using ElfShdr = ElfW(Shdr);
|
||||
using ElfNhdr = ElfW(Nhdr);
|
||||
using ElfSym = ElfW(Sym);
|
||||
|
||||
|
||||
@ -53,12 +54,18 @@ public:
|
||||
const char * end() const { return mapped + elf_size; }
|
||||
size_t size() const { return elf_size; }
|
||||
|
||||
/// Obtain build id from PT_NOTES section of program headers. Return empty string if does not exist.
|
||||
/// The string is returned in binary. Note that "readelf -n ./clickhouse-server" prints it in hex.
|
||||
String getBuildID() const;
|
||||
static String getBuildID(const char * nhdr_pos, size_t size);
|
||||
|
||||
private:
|
||||
MMapReadBufferFromFile in;
|
||||
size_t elf_size;
|
||||
const char * mapped;
|
||||
const ElfEhdr * header;
|
||||
const ElfShdr * section_headers;
|
||||
const ElfPhdr * program_headers;
|
||||
const char * section_names = nullptr;
|
||||
};
|
||||
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Field.h>
|
||||
#include <Core/AccurateComparison.h>
|
||||
#include <common/demangle.h>
|
||||
|
||||
|
||||
@ -14,7 +13,6 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_CONVERT_TYPE;
|
||||
extern const int BAD_TYPE_OF_FIELD;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
@ -177,243 +175,6 @@ template <> constexpr bool isDecimalField<DecimalField<Decimal64>>() { return tr
|
||||
template <> constexpr bool isDecimalField<DecimalField<Decimal128>>() { return true; }
|
||||
|
||||
|
||||
/** More precise comparison, used for index.
|
||||
* Differs from Field::operator< and Field::operator== in that it also compares values of different types.
|
||||
* Comparison rules are same as in FunctionsComparison (to be consistent with expression evaluation in query).
|
||||
*/
|
||||
class FieldVisitorAccurateEquals : public StaticVisitor<bool>
|
||||
{
|
||||
public:
|
||||
bool operator() (const UInt64 &, const Null &) const { return false; }
|
||||
bool operator() (const UInt64 & l, const UInt64 & r) const { return l == r; }
|
||||
bool operator() (const UInt64 & l, const UInt128 & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const UInt64 & l, const Int64 & r) const { return accurate::equalsOp(l, r); }
|
||||
bool operator() (const UInt64 & l, const Float64 & r) const { return accurate::equalsOp(l, r); }
|
||||
bool operator() (const UInt64 & l, const String & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const UInt64 & l, const Array & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const UInt64 & l, const Tuple & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const UInt64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); }
|
||||
|
||||
bool operator() (const Int64 &, const Null &) const { return false; }
|
||||
bool operator() (const Int64 & l, const UInt64 & r) const { return accurate::equalsOp(l, r); }
|
||||
bool operator() (const Int64 & l, const UInt128 & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Int64 & l, const Int64 & r) const { return l == r; }
|
||||
bool operator() (const Int64 & l, const Float64 & r) const { return accurate::equalsOp(l, r); }
|
||||
bool operator() (const Int64 & l, const String & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Int64 & l, const Array & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Int64 & l, const Tuple & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Int64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); }
|
||||
|
||||
bool operator() (const Float64 &, const Null &) const { return false; }
|
||||
bool operator() (const Float64 & l, const UInt64 & r) const { return accurate::equalsOp(l, r); }
|
||||
bool operator() (const Float64 & l, const UInt128 & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Float64 & l, const Int64 & r) const { return accurate::equalsOp(l, r); }
|
||||
bool operator() (const Float64 & l, const Float64 & r) const { return l == r; }
|
||||
bool operator() (const Float64 & l, const String & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Float64 & l, const Array & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Float64 & l, const Tuple & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Float64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); }
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const Null &, const T &) const
|
||||
{
|
||||
return std::is_same_v<T, Null>;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const String & l, const T & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, String>)
|
||||
return l == r;
|
||||
if constexpr (std::is_same_v<T, UInt128>)
|
||||
return stringToUUID(l) == r;
|
||||
if constexpr (std::is_same_v<T, Null>)
|
||||
return false;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const UInt128 & l, const T & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, UInt128>)
|
||||
return l == r;
|
||||
if constexpr (std::is_same_v<T, String>)
|
||||
return l == stringToUUID(r);
|
||||
if constexpr (std::is_same_v<T, Null>)
|
||||
return false;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const Array & l, const T & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, Array>)
|
||||
return l == r;
|
||||
if constexpr (std::is_same_v<T, Null>)
|
||||
return false;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const Tuple & l, const T & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, Tuple>)
|
||||
return l == r;
|
||||
if constexpr (std::is_same_v<T, Null>)
|
||||
return false;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
template <typename T, typename U>
|
||||
bool operator() (const DecimalField<T> & l, const U & r) const
|
||||
{
|
||||
if constexpr (isDecimalField<U>())
|
||||
return l == r;
|
||||
if constexpr (std::is_same_v<U, Int64> || std::is_same_v<U, UInt64>)
|
||||
return l == DecimalField<Decimal128>(r, 0);
|
||||
if constexpr (std::is_same_v<U, Null>)
|
||||
return false;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
template <typename T> bool operator() (const UInt64 & l, const DecimalField<T> & r) const { return DecimalField<Decimal128>(l, 0) == r; }
|
||||
template <typename T> bool operator() (const Int64 & l, const DecimalField<T> & r) const { return DecimalField<Decimal128>(l, 0) == r; }
|
||||
template <typename T> bool operator() (const Float64 & l, const DecimalField<T> & r) const { return cantCompare(l, r); }
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const AggregateFunctionStateData & l, const T & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, AggregateFunctionStateData>)
|
||||
return l == r;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename T, typename U>
|
||||
bool cantCompare(const T &, const U &) const
|
||||
{
|
||||
if constexpr (std::is_same_v<U, Null>)
|
||||
return false;
|
||||
throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()),
|
||||
ErrorCodes::BAD_TYPE_OF_FIELD);
|
||||
}
|
||||
};
|
||||
|
||||
class FieldVisitorAccurateLess : public StaticVisitor<bool>
|
||||
{
|
||||
public:
|
||||
bool operator() (const UInt64 &, const Null &) const { return false; }
|
||||
bool operator() (const UInt64 & l, const UInt64 & r) const { return l < r; }
|
||||
bool operator() (const UInt64 & l, const UInt128 & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const UInt64 & l, const Int64 & r) const { return accurate::lessOp(l, r); }
|
||||
bool operator() (const UInt64 & l, const Float64 & r) const { return accurate::lessOp(l, r); }
|
||||
bool operator() (const UInt64 & l, const String & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const UInt64 & l, const Array & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const UInt64 & l, const Tuple & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const UInt64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); }
|
||||
|
||||
bool operator() (const Int64 &, const Null &) const { return false; }
|
||||
bool operator() (const Int64 & l, const UInt64 & r) const { return accurate::lessOp(l, r); }
|
||||
bool operator() (const Int64 & l, const UInt128 & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Int64 & l, const Int64 & r) const { return l < r; }
|
||||
bool operator() (const Int64 & l, const Float64 & r) const { return accurate::lessOp(l, r); }
|
||||
bool operator() (const Int64 & l, const String & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Int64 & l, const Array & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Int64 & l, const Tuple & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Int64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); }
|
||||
|
||||
bool operator() (const Float64 &, const Null &) const { return false; }
|
||||
bool operator() (const Float64 & l, const UInt64 & r) const { return accurate::lessOp(l, r); }
|
||||
bool operator() (const Float64 & l, const UInt128 & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Float64 & l, const Int64 & r) const { return accurate::lessOp(l, r); }
|
||||
bool operator() (const Float64 & l, const Float64 & r) const { return l < r; }
|
||||
bool operator() (const Float64 & l, const String & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Float64 & l, const Array & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Float64 & l, const Tuple & r) const { return cantCompare(l, r); }
|
||||
bool operator() (const Float64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); }
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const Null &, const T &) const
|
||||
{
|
||||
return !std::is_same_v<T, Null>;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const String & l, const T & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, String>)
|
||||
return l < r;
|
||||
if constexpr (std::is_same_v<T, UInt128>)
|
||||
return stringToUUID(l) < r;
|
||||
if constexpr (std::is_same_v<T, Null>)
|
||||
return false;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const UInt128 & l, const T & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, UInt128>)
|
||||
return l < r;
|
||||
if constexpr (std::is_same_v<T, String>)
|
||||
return l < stringToUUID(r);
|
||||
if constexpr (std::is_same_v<T, Null>)
|
||||
return false;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const Array & l, const T & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, Array>)
|
||||
return l < r;
|
||||
if constexpr (std::is_same_v<T, Null>)
|
||||
return false;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const Tuple & l, const T & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, Tuple>)
|
||||
return l < r;
|
||||
if constexpr (std::is_same_v<T, Null>)
|
||||
return false;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
template <typename T, typename U>
|
||||
bool operator() (const DecimalField<T> & l, const U & r) const
|
||||
{
|
||||
if constexpr (isDecimalField<U>())
|
||||
return l < r;
|
||||
if constexpr (std::is_same_v<U, Int64> || std::is_same_v<U, UInt64>)
|
||||
return l < DecimalField<Decimal128>(r, 0);
|
||||
if constexpr (std::is_same_v<U, Null>)
|
||||
return false;
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
template <typename T> bool operator() (const UInt64 & l, const DecimalField<T> & r) const { return DecimalField<Decimal128>(l, 0) < r; }
|
||||
template <typename T> bool operator() (const Int64 & l, const DecimalField<T> & r) const { return DecimalField<Decimal128>(l, 0) < r; }
|
||||
template <typename T> bool operator() (const Float64 &, const DecimalField<T> &) const { return false; }
|
||||
|
||||
template <typename T>
|
||||
bool operator() (const AggregateFunctionStateData & l, const T & r) const
|
||||
{
|
||||
return cantCompare(l, r);
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename T, typename U>
|
||||
bool cantCompare(const T &, const U &) const
|
||||
{
|
||||
throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()),
|
||||
ErrorCodes::BAD_TYPE_OF_FIELD);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/** Implements `+=` operation.
|
||||
* Returns false if the result is zero.
|
||||
*/
|
||||
|
||||
142
src/Common/FieldVisitorsAccurateComparison.h
Normal file
142
src/Common/FieldVisitorsAccurateComparison.h
Normal file
@ -0,0 +1,142 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Field.h>
|
||||
#include <Core/AccurateComparison.h>
|
||||
#include <common/demangle.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int BAD_TYPE_OF_FIELD;
|
||||
}
|
||||
|
||||
/** More precise comparison, used for index.
|
||||
* Differs from Field::operator< and Field::operator== in that it also compares values of different types.
|
||||
* Comparison rules are same as in FunctionsComparison (to be consistent with expression evaluation in query).
|
||||
*/
|
||||
class FieldVisitorAccurateEquals : public StaticVisitor<bool>
|
||||
{
|
||||
public:
|
||||
template <typename T, typename U>
|
||||
bool operator() (const T & l, const U & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, Null> || std::is_same_v<U, Null>)
|
||||
return std::is_same_v<T, U>;
|
||||
else
|
||||
{
|
||||
if constexpr (std::is_same_v<T, U>)
|
||||
return l == r;
|
||||
|
||||
if constexpr (std::is_arithmetic_v<T> && std::is_arithmetic_v<U>)
|
||||
return accurate::equalsOp(l, r);
|
||||
|
||||
if constexpr (isDecimalField<T>() && isDecimalField<U>())
|
||||
return l == r;
|
||||
|
||||
if constexpr (isDecimalField<T>() && std::is_arithmetic_v<U>)
|
||||
return l == DecimalField<Decimal128>(r, 0);
|
||||
|
||||
if constexpr (std::is_arithmetic_v<T> && isDecimalField<U>())
|
||||
return DecimalField<Decimal128>(l, 0) == r;
|
||||
|
||||
if constexpr (std::is_same_v<T, String>)
|
||||
{
|
||||
if constexpr (std::is_same_v<U, UInt128>)
|
||||
return stringToUUID(l) == r;
|
||||
|
||||
if constexpr (std::is_arithmetic_v<U>)
|
||||
{
|
||||
ReadBufferFromString in(l);
|
||||
T parsed;
|
||||
readText(parsed, in);
|
||||
return operator()(parsed, r);
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (std::is_same_v<U, String>)
|
||||
{
|
||||
if constexpr (std::is_same_v<T, UInt128>)
|
||||
return l == stringToUUID(r);
|
||||
|
||||
if constexpr (std::is_arithmetic_v<T>)
|
||||
{
|
||||
ReadBufferFromString in(r);
|
||||
T parsed;
|
||||
readText(parsed, in);
|
||||
return operator()(l, parsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()),
|
||||
ErrorCodes::BAD_TYPE_OF_FIELD);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class FieldVisitorAccurateLess : public StaticVisitor<bool>
|
||||
{
|
||||
public:
|
||||
template <typename T, typename U>
|
||||
bool operator() (const T & l, const U & r) const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, Null> || std::is_same_v<U, Null>)
|
||||
return false;
|
||||
else
|
||||
{
|
||||
if constexpr (std::is_same_v<T, U>)
|
||||
return l < r;
|
||||
|
||||
if constexpr (std::is_arithmetic_v<T> && std::is_arithmetic_v<U>)
|
||||
return accurate::lessOp(l, r);
|
||||
|
||||
if constexpr (isDecimalField<T>() && isDecimalField<U>())
|
||||
return l < r;
|
||||
|
||||
if constexpr (isDecimalField<T>() && std::is_arithmetic_v<U>)
|
||||
return l < DecimalField<Decimal128>(r, 0);
|
||||
|
||||
if constexpr (std::is_arithmetic_v<T> && isDecimalField<U>())
|
||||
return DecimalField<Decimal128>(l, 0) < r;
|
||||
|
||||
if constexpr (std::is_same_v<T, String>)
|
||||
{
|
||||
if constexpr (std::is_same_v<U, UInt128>)
|
||||
return stringToUUID(l) < r;
|
||||
|
||||
if constexpr (std::is_arithmetic_v<U>)
|
||||
{
|
||||
ReadBufferFromString in(l);
|
||||
T parsed;
|
||||
readText(parsed, in);
|
||||
return operator()(parsed, r);
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (std::is_same_v<U, String>)
|
||||
{
|
||||
if constexpr (std::is_same_v<T, UInt128>)
|
||||
return l < stringToUUID(r);
|
||||
|
||||
if constexpr (std::is_arithmetic_v<T>)
|
||||
{
|
||||
ReadBufferFromString in(r);
|
||||
T parsed;
|
||||
readText(parsed, in);
|
||||
return operator()(l, parsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()),
|
||||
ErrorCodes::BAD_TYPE_OF_FIELD);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
@ -38,6 +38,7 @@ void OptimizedRegularExpressionImpl<thread_safe>::analyze(
|
||||
required_substring_is_prefix = false;
|
||||
required_substring.clear();
|
||||
bool has_alternative_on_depth_0 = false;
|
||||
bool has_case_insensitive_flag = false;
|
||||
|
||||
/// Substring with a position.
|
||||
using Substring = std::pair<std::string, size_t>;
|
||||
@ -65,7 +66,17 @@ void OptimizedRegularExpressionImpl<thread_safe>::analyze(
|
||||
|
||||
switch (*pos)
|
||||
{
|
||||
case '|': case '(': case ')': case '^': case '$': case '.': case '[': case '?': case '*': case '+': case '{':
|
||||
case '|':
|
||||
case '(':
|
||||
case ')':
|
||||
case '^':
|
||||
case '$':
|
||||
case '.':
|
||||
case '[':
|
||||
case '?':
|
||||
case '*':
|
||||
case '+':
|
||||
case '{':
|
||||
if (depth == 0 && !in_curly_braces && !in_square_braces)
|
||||
{
|
||||
if (last_substring->first.empty())
|
||||
@ -110,6 +121,28 @@ void OptimizedRegularExpressionImpl<thread_safe>::analyze(
|
||||
trivial_substrings.resize(trivial_substrings.size() + 1);
|
||||
last_substring = &trivial_substrings.back();
|
||||
}
|
||||
|
||||
/// Check for case-insensitive flag.
|
||||
if (pos + 1 < end && pos[1] == '?')
|
||||
{
|
||||
for (size_t offset = 2; pos + offset < end; ++offset)
|
||||
{
|
||||
if (pos[offset] == '-' /// it means flag negation
|
||||
/// various possible flags, actually only imsU are supported by re2
|
||||
|| (pos[offset] >= 'a' && pos[offset] <= 'z')
|
||||
|| (pos[offset] >= 'A' && pos[offset] <= 'Z'))
|
||||
{
|
||||
if (pos[offset] == 'i')
|
||||
{
|
||||
/// Actually it can be negated case-insensitive flag. But we don't care.
|
||||
has_case_insensitive_flag = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
++pos;
|
||||
break;
|
||||
@ -209,7 +242,7 @@ void OptimizedRegularExpressionImpl<thread_safe>::analyze(
|
||||
|
||||
if (!is_trivial)
|
||||
{
|
||||
if (!has_alternative_on_depth_0)
|
||||
if (!has_alternative_on_depth_0 && !has_case_insensitive_flag)
|
||||
{
|
||||
/// We choose the non-alternative substring of the maximum length for first search.
|
||||
|
||||
|
||||
@ -196,6 +196,20 @@ void collectSymbolsFromProgramHeaders(dl_phdr_info * info,
|
||||
}
|
||||
|
||||
|
||||
String getBuildIDFromProgramHeaders(dl_phdr_info * info)
|
||||
{
|
||||
for (size_t header_index = 0; header_index < info->dlpi_phnum; ++header_index)
|
||||
{
|
||||
const ElfPhdr & phdr = info->dlpi_phdr[header_index];
|
||||
if (phdr.p_type != PT_NOTE)
|
||||
continue;
|
||||
|
||||
return Elf::getBuildID(reinterpret_cast<const char *>(info->dlpi_addr + phdr.p_vaddr), phdr.p_memsz);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
void collectSymbolsFromELFSymbolTable(
|
||||
dl_phdr_info * info,
|
||||
const Elf & elf,
|
||||
@ -283,8 +297,31 @@ void collectSymbolsFromELF(dl_phdr_info * info,
|
||||
|
||||
object_name = std::filesystem::exists(debug_info_path) ? debug_info_path : canonical_path;
|
||||
|
||||
/// But we have to compare Build ID to check that debug info corresponds to the same executable.
|
||||
String our_build_id = getBuildIDFromProgramHeaders(info);
|
||||
|
||||
SymbolIndex::Object object;
|
||||
object.elf = std::make_unique<Elf>(object_name);
|
||||
|
||||
String file_build_id = object.elf->getBuildID();
|
||||
|
||||
if (our_build_id != file_build_id)
|
||||
{
|
||||
/// If debug info doesn't correspond to our binary, fallback to the info in our binary.
|
||||
if (object_name != canonical_path)
|
||||
{
|
||||
object_name = canonical_path;
|
||||
object.elf = std::make_unique<Elf>(object_name);
|
||||
|
||||
/// But it can still be outdated, for example, if executable file was deleted from filesystem and replaced by another file.
|
||||
file_build_id = object.elf->getBuildID();
|
||||
if (our_build_id != file_build_id)
|
||||
return;
|
||||
}
|
||||
else
|
||||
return;
|
||||
}
|
||||
|
||||
object.address_begin = reinterpret_cast<const void *>(info->dlpi_addr);
|
||||
object.address_end = reinterpret_cast<const void *>(info->dlpi_addr + object.elf->size());
|
||||
object.name = object_name;
|
||||
|
||||
@ -23,7 +23,7 @@ namespace ProfileEvents
|
||||
namespace Coordination
|
||||
{
|
||||
|
||||
Exception::Exception(const std::string & msg, const int32_t code_, int)
|
||||
Exception::Exception(const std::string & msg, const Error code_, int)
|
||||
: DB::Exception(msg, DB::ErrorCodes::KEEPER_EXCEPTION), code(code_)
|
||||
{
|
||||
if (Coordination::isUserError(code))
|
||||
@ -34,17 +34,17 @@ Exception::Exception(const std::string & msg, const int32_t code_, int)
|
||||
ProfileEvents::increment(ProfileEvents::ZooKeeperOtherExceptions);
|
||||
}
|
||||
|
||||
Exception::Exception(const std::string & msg, const int32_t code_)
|
||||
Exception::Exception(const std::string & msg, const Error code_)
|
||||
: Exception(msg + " (" + errorMessage(code_) + ")", code_, 0)
|
||||
{
|
||||
}
|
||||
|
||||
Exception::Exception(const int32_t code_)
|
||||
Exception::Exception(const Error code_)
|
||||
: Exception(errorMessage(code_), code_, 0)
|
||||
{
|
||||
}
|
||||
|
||||
Exception::Exception(const int32_t code_, const std::string & path)
|
||||
Exception::Exception(const Error code_, const std::string & path)
|
||||
: Exception(std::string{errorMessage(code_)} + ", path: " + path, code_, 0)
|
||||
{
|
||||
}
|
||||
@ -58,10 +58,10 @@ using namespace DB;
|
||||
static void addRootPath(String & path, const String & root_path)
|
||||
{
|
||||
if (path.empty())
|
||||
throw Exception("Path cannot be empty", ZBADARGUMENTS);
|
||||
throw Exception("Path cannot be empty", Error::ZBADARGUMENTS);
|
||||
|
||||
if (path[0] != '/')
|
||||
throw Exception("Path must begin with /", ZBADARGUMENTS);
|
||||
throw Exception("Path must begin with /", Error::ZBADARGUMENTS);
|
||||
|
||||
if (root_path.empty())
|
||||
return;
|
||||
@ -78,64 +78,62 @@ static void removeRootPath(String & path, const String & root_path)
|
||||
return;
|
||||
|
||||
if (path.size() <= root_path.size())
|
||||
throw Exception("Received path is not longer than root_path", ZDATAINCONSISTENCY);
|
||||
throw Exception("Received path is not longer than root_path", Error::ZDATAINCONSISTENCY);
|
||||
|
||||
path = path.substr(root_path.size());
|
||||
}
|
||||
|
||||
|
||||
const char * errorMessage(int32_t code)
|
||||
const char * errorMessage(Error code)
|
||||
{
|
||||
switch (code)
|
||||
{
|
||||
case ZOK: return "Ok";
|
||||
case ZSYSTEMERROR: return "System error";
|
||||
case ZRUNTIMEINCONSISTENCY: return "Run time inconsistency";
|
||||
case ZDATAINCONSISTENCY: return "Data inconsistency";
|
||||
case ZCONNECTIONLOSS: return "Connection loss";
|
||||
case ZMARSHALLINGERROR: return "Marshalling error";
|
||||
case ZUNIMPLEMENTED: return "Unimplemented";
|
||||
case ZOPERATIONTIMEOUT: return "Operation timeout";
|
||||
case ZBADARGUMENTS: return "Bad arguments";
|
||||
case ZINVALIDSTATE: return "Invalid zhandle state";
|
||||
case ZAPIERROR: return "API error";
|
||||
case ZNONODE: return "No node";
|
||||
case ZNOAUTH: return "Not authenticated";
|
||||
case ZBADVERSION: return "Bad version";
|
||||
case ZNOCHILDRENFOREPHEMERALS: return "No children for ephemerals";
|
||||
case ZNODEEXISTS: return "Node exists";
|
||||
case ZNOTEMPTY: return "Not empty";
|
||||
case ZSESSIONEXPIRED: return "Session expired";
|
||||
case ZINVALIDCALLBACK: return "Invalid callback";
|
||||
case ZINVALIDACL: return "Invalid ACL";
|
||||
case ZAUTHFAILED: return "Authentication failed";
|
||||
case ZCLOSING: return "ZooKeeper is closing";
|
||||
case ZNOTHING: return "(not error) no server responses to process";
|
||||
case ZSESSIONMOVED: return "Session moved to another server, so operation is ignored";
|
||||
case Error::ZOK: return "Ok";
|
||||
case Error::ZSYSTEMERROR: return "System error";
|
||||
case Error::ZRUNTIMEINCONSISTENCY: return "Run time inconsistency";
|
||||
case Error::ZDATAINCONSISTENCY: return "Data inconsistency";
|
||||
case Error::ZCONNECTIONLOSS: return "Connection loss";
|
||||
case Error::ZMARSHALLINGERROR: return "Marshalling error";
|
||||
case Error::ZUNIMPLEMENTED: return "Unimplemented";
|
||||
case Error::ZOPERATIONTIMEOUT: return "Operation timeout";
|
||||
case Error::ZBADARGUMENTS: return "Bad arguments";
|
||||
case Error::ZINVALIDSTATE: return "Invalid zhandle state";
|
||||
case Error::ZAPIERROR: return "API error";
|
||||
case Error::ZNONODE: return "No node";
|
||||
case Error::ZNOAUTH: return "Not authenticated";
|
||||
case Error::ZBADVERSION: return "Bad version";
|
||||
case Error::ZNOCHILDRENFOREPHEMERALS: return "No children for ephemerals";
|
||||
case Error::ZNODEEXISTS: return "Node exists";
|
||||
case Error::ZNOTEMPTY: return "Not empty";
|
||||
case Error::ZSESSIONEXPIRED: return "Session expired";
|
||||
case Error::ZINVALIDCALLBACK: return "Invalid callback";
|
||||
case Error::ZINVALIDACL: return "Invalid ACL";
|
||||
case Error::ZAUTHFAILED: return "Authentication failed";
|
||||
case Error::ZCLOSING: return "ZooKeeper is closing";
|
||||
case Error::ZNOTHING: return "(not error) no server responses to process";
|
||||
case Error::ZSESSIONMOVED: return "Session moved to another server, so operation is ignored";
|
||||
}
|
||||
if (code > 0)
|
||||
return strerror(code);
|
||||
|
||||
return "unknown error";
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
bool isHardwareError(int32_t zk_return_code)
|
||||
bool isHardwareError(Error zk_return_code)
|
||||
{
|
||||
return zk_return_code == ZINVALIDSTATE
|
||||
|| zk_return_code == ZSESSIONEXPIRED
|
||||
|| zk_return_code == ZSESSIONMOVED
|
||||
|| zk_return_code == ZCONNECTIONLOSS
|
||||
|| zk_return_code == ZMARSHALLINGERROR
|
||||
|| zk_return_code == ZOPERATIONTIMEOUT;
|
||||
return zk_return_code == Error::ZINVALIDSTATE
|
||||
|| zk_return_code == Error::ZSESSIONEXPIRED
|
||||
|| zk_return_code == Error::ZSESSIONMOVED
|
||||
|| zk_return_code == Error::ZCONNECTIONLOSS
|
||||
|| zk_return_code == Error::ZMARSHALLINGERROR
|
||||
|| zk_return_code == Error::ZOPERATIONTIMEOUT;
|
||||
}
|
||||
|
||||
bool isUserError(int32_t zk_return_code)
|
||||
bool isUserError(Error zk_return_code)
|
||||
{
|
||||
return zk_return_code == ZNONODE
|
||||
|| zk_return_code == ZBADVERSION
|
||||
|| zk_return_code == ZNOCHILDRENFOREPHEMERALS
|
||||
|| zk_return_code == ZNODEEXISTS
|
||||
|| zk_return_code == ZNOTEMPTY;
|
||||
return zk_return_code == Error::ZNONODE
|
||||
|| zk_return_code == Error::ZBADVERSION
|
||||
|| zk_return_code == Error::ZNOCHILDRENFOREPHEMERALS
|
||||
|| zk_return_code == Error::ZNODEEXISTS
|
||||
|| zk_return_code == Error::ZNOTEMPTY;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -53,6 +53,57 @@ struct Stat
|
||||
int64_t pzxid;
|
||||
};
|
||||
|
||||
enum class Error : int32_t
|
||||
{
|
||||
ZOK = 0,
|
||||
|
||||
/** System and server-side errors.
|
||||
* This is never thrown by the server, it shouldn't be used other than
|
||||
* to indicate a range. Specifically error codes greater than this
|
||||
* value, but lesser than ZAPIERROR, are system errors.
|
||||
*/
|
||||
ZSYSTEMERROR = -1,
|
||||
|
||||
ZRUNTIMEINCONSISTENCY = -2, /// A runtime inconsistency was found
|
||||
ZDATAINCONSISTENCY = -3, /// A data inconsistency was found
|
||||
ZCONNECTIONLOSS = -4, /// Connection to the server has been lost
|
||||
ZMARSHALLINGERROR = -5, /// Error while marshalling or unmarshalling data
|
||||
ZUNIMPLEMENTED = -6, /// Operation is unimplemented
|
||||
ZOPERATIONTIMEOUT = -7, /// Operation timeout
|
||||
ZBADARGUMENTS = -8, /// Invalid arguments
|
||||
ZINVALIDSTATE = -9, /// Invliad zhandle state
|
||||
|
||||
/** API errors.
|
||||
* This is never thrown by the server, it shouldn't be used other than
|
||||
* to indicate a range. Specifically error codes greater than this
|
||||
* value are API errors.
|
||||
*/
|
||||
ZAPIERROR = -100,
|
||||
|
||||
ZNONODE = -101, /// Node does not exist
|
||||
ZNOAUTH = -102, /// Not authenticated
|
||||
ZBADVERSION = -103, /// Version conflict
|
||||
ZNOCHILDRENFOREPHEMERALS = -108, /// Ephemeral nodes may not have children
|
||||
ZNODEEXISTS = -110, /// The node already exists
|
||||
ZNOTEMPTY = -111, /// The node has children
|
||||
ZSESSIONEXPIRED = -112, /// The session has been expired by the server
|
||||
ZINVALIDCALLBACK = -113, /// Invalid callback specified
|
||||
ZINVALIDACL = -114, /// Invalid ACL specified
|
||||
ZAUTHFAILED = -115, /// Client authentication failed
|
||||
ZCLOSING = -116, /// ZooKeeper is closing
|
||||
ZNOTHING = -117, /// (not error) no server responses to process
|
||||
ZSESSIONMOVED = -118 /// Session moved to another server, so operation is ignored
|
||||
};
|
||||
|
||||
/// Network errors and similar. You should reinitialize ZooKeeper session in case of these errors
|
||||
bool isHardwareError(Error code);
|
||||
|
||||
/// Valid errors sent from the server about database state (like "no node"). Logical and authentication errors (like "bad arguments") are not here.
|
||||
bool isUserError(Error code);
|
||||
|
||||
const char * errorMessage(Error code);
|
||||
|
||||
|
||||
struct Request;
|
||||
using RequestPtr = std::shared_ptr<Request>;
|
||||
using Requests = std::vector<RequestPtr>;
|
||||
@ -74,7 +125,7 @@ using ResponseCallback = std::function<void(const Response &)>;
|
||||
|
||||
struct Response
|
||||
{
|
||||
int32_t error = 0;
|
||||
Error error = Error::ZOK;
|
||||
Response() = default;
|
||||
Response(const Response &) = default;
|
||||
Response & operator=(const Response &) = default;
|
||||
@ -225,56 +276,6 @@ using CheckCallback = std::function<void(const CheckResponse &)>;
|
||||
using MultiCallback = std::function<void(const MultiResponse &)>;
|
||||
|
||||
|
||||
enum Error
|
||||
{
|
||||
ZOK = 0,
|
||||
|
||||
/** System and server-side errors.
|
||||
* This is never thrown by the server, it shouldn't be used other than
|
||||
* to indicate a range. Specifically error codes greater than this
|
||||
* value, but lesser than ZAPIERROR, are system errors.
|
||||
*/
|
||||
ZSYSTEMERROR = -1,
|
||||
|
||||
ZRUNTIMEINCONSISTENCY = -2, /// A runtime inconsistency was found
|
||||
ZDATAINCONSISTENCY = -3, /// A data inconsistency was found
|
||||
ZCONNECTIONLOSS = -4, /// Connection to the server has been lost
|
||||
ZMARSHALLINGERROR = -5, /// Error while marshalling or unmarshalling data
|
||||
ZUNIMPLEMENTED = -6, /// Operation is unimplemented
|
||||
ZOPERATIONTIMEOUT = -7, /// Operation timeout
|
||||
ZBADARGUMENTS = -8, /// Invalid arguments
|
||||
ZINVALIDSTATE = -9, /// Invliad zhandle state
|
||||
|
||||
/** API errors.
|
||||
* This is never thrown by the server, it shouldn't be used other than
|
||||
* to indicate a range. Specifically error codes greater than this
|
||||
* value are API errors.
|
||||
*/
|
||||
ZAPIERROR = -100,
|
||||
|
||||
ZNONODE = -101, /// Node does not exist
|
||||
ZNOAUTH = -102, /// Not authenticated
|
||||
ZBADVERSION = -103, /// Version conflict
|
||||
ZNOCHILDRENFOREPHEMERALS = -108, /// Ephemeral nodes may not have children
|
||||
ZNODEEXISTS = -110, /// The node already exists
|
||||
ZNOTEMPTY = -111, /// The node has children
|
||||
ZSESSIONEXPIRED = -112, /// The session has been expired by the server
|
||||
ZINVALIDCALLBACK = -113, /// Invalid callback specified
|
||||
ZINVALIDACL = -114, /// Invalid ACL specified
|
||||
ZAUTHFAILED = -115, /// Client authentication failed
|
||||
ZCLOSING = -116, /// ZooKeeper is closing
|
||||
ZNOTHING = -117, /// (not error) no server responses to process
|
||||
ZSESSIONMOVED = -118 /// Session moved to another server, so operation is ignored
|
||||
};
|
||||
|
||||
/// Network errors and similar. You should reinitialize ZooKeeper session in case of these errors
|
||||
bool isHardwareError(int32_t code);
|
||||
|
||||
/// Valid errors sent from the server about database state (like "no node"). Logical and authentication errors (like "bad arguments") are not here.
|
||||
bool isUserError(int32_t code);
|
||||
|
||||
const char * errorMessage(int32_t code);
|
||||
|
||||
/// For watches.
|
||||
enum State
|
||||
{
|
||||
@ -301,19 +302,19 @@ class Exception : public DB::Exception
|
||||
{
|
||||
private:
|
||||
/// Delegate constructor, used to minimize repetition; last parameter used for overload resolution.
|
||||
Exception(const std::string & msg, const int32_t code_, int);
|
||||
Exception(const std::string & msg, const Error code_, int);
|
||||
|
||||
public:
|
||||
explicit Exception(const int32_t code_);
|
||||
Exception(const std::string & msg, const int32_t code_);
|
||||
Exception(const int32_t code_, const std::string & path);
|
||||
explicit Exception(const Error code_);
|
||||
Exception(const std::string & msg, const Error code_);
|
||||
Exception(const Error code_, const std::string & path);
|
||||
Exception(const Exception & exc);
|
||||
|
||||
const char * name() const throw() override { return "Coordination::Exception"; }
|
||||
const char * className() const throw() override { return "Coordination::Exception"; }
|
||||
Exception * clone() const override { return new Exception(*this); }
|
||||
|
||||
const int32_t code;
|
||||
const Error code;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -29,11 +29,11 @@ public:
|
||||
if (zookeeper->tryGet(path, result_str, &stat))
|
||||
{
|
||||
result = std::stol(result_str) + 1;
|
||||
success = zookeeper->trySet(path, std::to_string(result), stat.version) == Coordination::ZOK;
|
||||
success = zookeeper->trySet(path, std::to_string(result), stat.version) == Coordination::Error::ZOK;
|
||||
}
|
||||
else
|
||||
{
|
||||
success = zookeeper->tryCreate(path, std::to_string(result), zkutil::CreateMode::Persistent) == Coordination::ZOK;
|
||||
success = zookeeper->tryCreate(path, std::to_string(result), zkutil::CreateMode::Persistent) == Coordination::Error::ZOK;
|
||||
}
|
||||
}
|
||||
while (!success);
|
||||
|
||||
@ -21,12 +21,12 @@ public:
|
||||
|
||||
/// If it is user error throws KeeperMultiException else throws ordinary KeeperException
|
||||
/// If it is ZOK does nothing
|
||||
static void check(int32_t code, const Coordination::Requests & requests, const Coordination::Responses & responses);
|
||||
static void check(Coordination::Error code, const Coordination::Requests & requests, const Coordination::Responses & responses);
|
||||
|
||||
KeeperMultiException(int32_t code, const Coordination::Requests & requests, const Coordination::Responses & responses);
|
||||
KeeperMultiException(Coordination::Error code, const Coordination::Requests & requests, const Coordination::Responses & responses);
|
||||
|
||||
private:
|
||||
static size_t getFailedOpIndex(int32_t code, const Coordination::Responses & responses);
|
||||
static size_t getFailedOpIndex(Coordination::Error code, const Coordination::Responses & responses);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -36,7 +36,12 @@ public:
|
||||
* It means that different participants of leader election have different identifiers
|
||||
* and existence of more than one ephemeral node with same identifier indicates an error.
|
||||
*/
|
||||
LeaderElection(DB::BackgroundSchedulePool & pool_, const std::string & path_, ZooKeeper & zookeeper_, LeadershipHandler handler_, const std::string & identifier_ = "")
|
||||
LeaderElection(
|
||||
DB::BackgroundSchedulePool & pool_,
|
||||
const std::string & path_,
|
||||
ZooKeeper & zookeeper_,
|
||||
LeadershipHandler handler_,
|
||||
const std::string & identifier_)
|
||||
: pool(pool_), path(path_), zookeeper(zookeeper_), handler(handler_), identifier(identifier_)
|
||||
, log_name("LeaderElection (" + path + ")")
|
||||
, log(&Poco::Logger::get(log_name))
|
||||
@ -121,7 +126,7 @@ private:
|
||||
{
|
||||
DB::tryLogCurrentException(log);
|
||||
|
||||
if (e.code == Coordination::ZSESSIONEXPIRED)
|
||||
if (e.code == Coordination::Error::ZSESSIONEXPIRED)
|
||||
return;
|
||||
}
|
||||
catch (...)
|
||||
|
||||
@ -16,13 +16,13 @@ bool Lock::tryLock()
|
||||
else
|
||||
{
|
||||
std::string dummy;
|
||||
int32_t code = zookeeper->tryCreate(lock_path, lock_message, zkutil::CreateMode::Ephemeral, dummy);
|
||||
Coordination::Error code = zookeeper->tryCreate(lock_path, lock_message, zkutil::CreateMode::Ephemeral, dummy);
|
||||
|
||||
if (code == Coordination::ZNODEEXISTS)
|
||||
if (code == Coordination::Error::ZNODEEXISTS)
|
||||
{
|
||||
locked.reset();
|
||||
}
|
||||
else if (code == Coordination::ZOK)
|
||||
else if (code == Coordination::Error::ZOK)
|
||||
{
|
||||
locked = std::make_unique<ZooKeeperHandler>(zookeeper);
|
||||
}
|
||||
|
||||
@ -158,7 +158,7 @@ struct TestKeeperMultiRequest final : MultiRequest, TestKeeperRequest
|
||||
requests.push_back(std::make_shared<TestKeeperCheckRequest>(*concrete_request_check));
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal command as part of multi ZooKeeper request", ZBADARGUMENTS);
|
||||
throw Exception("Illegal command as part of multi ZooKeeper request", Error::ZBADARGUMENTS);
|
||||
}
|
||||
}
|
||||
|
||||
@ -338,7 +338,7 @@ ResponsePtr TestKeeperListRequest::process(TestKeeper::Container & container, in
|
||||
{
|
||||
auto path_prefix = path;
|
||||
if (path_prefix.empty())
|
||||
throw Exception("Logical error: path cannot be empty", ZSESSIONEXPIRED);
|
||||
throw Exception("Logical error: path cannot be empty", Error::ZSESSIONEXPIRED);
|
||||
|
||||
if (path_prefix.back() != '/')
|
||||
path_prefix += '/';
|
||||
@ -514,7 +514,7 @@ void TestKeeper::finalize()
|
||||
WatchResponse response;
|
||||
response.type = SESSION;
|
||||
response.state = EXPIRED_SESSION;
|
||||
response.error = ZSESSIONEXPIRED;
|
||||
response.error = Error::ZSESSIONEXPIRED;
|
||||
|
||||
for (auto & callback : path_watch.second)
|
||||
{
|
||||
@ -541,7 +541,7 @@ void TestKeeper::finalize()
|
||||
if (info.callback)
|
||||
{
|
||||
ResponsePtr response = info.request->createResponse();
|
||||
response->error = ZSESSIONEXPIRED;
|
||||
response->error = Error::ZSESSIONEXPIRED;
|
||||
try
|
||||
{
|
||||
info.callback(*response);
|
||||
@ -556,7 +556,7 @@ void TestKeeper::finalize()
|
||||
WatchResponse response;
|
||||
response.type = SESSION;
|
||||
response.state = EXPIRED_SESSION;
|
||||
response.error = ZSESSIONEXPIRED;
|
||||
response.error = Error::ZSESSIONEXPIRED;
|
||||
try
|
||||
{
|
||||
info.watch(response);
|
||||
@ -587,10 +587,10 @@ void TestKeeper::pushRequest(RequestInfo && request)
|
||||
std::lock_guard lock(push_request_mutex);
|
||||
|
||||
if (expired)
|
||||
throw Exception("Session expired", ZSESSIONEXPIRED);
|
||||
throw Exception("Session expired", Error::ZSESSIONEXPIRED);
|
||||
|
||||
if (!requests_queue.tryPush(std::move(request), operation_timeout.totalMilliseconds()))
|
||||
throw Exception("Cannot push request to queue within operation timeout", ZOPERATIONTIMEOUT);
|
||||
throw Exception("Cannot push request to queue within operation timeout", Error::ZOPERATIONTIMEOUT);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
|
||||
@ -38,9 +38,9 @@ const int CreateMode::PersistentSequential = 2;
|
||||
const int CreateMode::EphemeralSequential = 3;
|
||||
|
||||
|
||||
static void check(int32_t code, const std::string & path)
|
||||
static void check(Coordination::Error code, const std::string & path)
|
||||
{
|
||||
if (code)
|
||||
if (code != Coordination::Error::ZOK)
|
||||
throw KeeperException(code, path);
|
||||
}
|
||||
|
||||
@ -59,7 +59,7 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho
|
||||
if (implementation == "zookeeper")
|
||||
{
|
||||
if (hosts.empty())
|
||||
throw KeeperException("No hosts passed to ZooKeeper constructor.", Coordination::ZBADARGUMENTS);
|
||||
throw KeeperException("No hosts passed to ZooKeeper constructor.", Coordination::Error::ZBADARGUMENTS);
|
||||
|
||||
std::vector<std::string> hosts_strings;
|
||||
splitInto<','>(hosts_strings, hosts);
|
||||
@ -84,7 +84,7 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho
|
||||
}
|
||||
|
||||
if (nodes.empty())
|
||||
throw KeeperException("Cannot use any of provided ZooKeeper nodes", Coordination::ZBADARGUMENTS);
|
||||
throw KeeperException("Cannot use any of provided ZooKeeper nodes", Coordination::Error::ZBADARGUMENTS);
|
||||
|
||||
impl = std::make_unique<Coordination::ZooKeeper>(
|
||||
nodes,
|
||||
@ -112,7 +112,7 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho
|
||||
}
|
||||
|
||||
if (!chroot.empty() && !exists("/"))
|
||||
throw KeeperException("Zookeeper root doesn't exist. You should create root node " + chroot + " before start.", Coordination::ZNONODE);
|
||||
throw KeeperException("Zookeeper root doesn't exist. You should create root node " + chroot + " before start.", Coordination::Error::ZNONODE);
|
||||
}
|
||||
|
||||
ZooKeeper::ZooKeeper(const std::string & hosts_, const std::string & identity_, int32_t session_timeout_ms_,
|
||||
@ -164,7 +164,7 @@ struct ZooKeeperArgs
|
||||
implementation = config.getString(config_name + "." + key);
|
||||
}
|
||||
else
|
||||
throw KeeperException(std::string("Unknown key ") + key + " in config file", Coordination::ZBADARGUMENTS);
|
||||
throw KeeperException(std::string("Unknown key ") + key + " in config file", Coordination::Error::ZBADARGUMENTS);
|
||||
}
|
||||
|
||||
/// Shuffle the hosts to distribute the load among ZooKeeper nodes.
|
||||
@ -182,7 +182,7 @@ struct ZooKeeperArgs
|
||||
if (!chroot.empty())
|
||||
{
|
||||
if (chroot.front() != '/')
|
||||
throw KeeperException(std::string("Root path in config file should start with '/', but got ") + chroot, Coordination::ZBADARGUMENTS);
|
||||
throw KeeperException(std::string("Root path in config file should start with '/', but got ") + chroot, Coordination::Error::ZBADARGUMENTS);
|
||||
if (chroot.back() == '/')
|
||||
chroot.pop_back();
|
||||
}
|
||||
@ -211,17 +211,17 @@ static Coordination::WatchCallback callbackForEvent(const EventPtr & watch)
|
||||
}
|
||||
|
||||
|
||||
int32_t ZooKeeper::getChildrenImpl(const std::string & path, Strings & res,
|
||||
Coordination::Error ZooKeeper::getChildrenImpl(const std::string & path, Strings & res,
|
||||
Coordination::Stat * stat,
|
||||
Coordination::WatchCallback watch_callback)
|
||||
{
|
||||
int32_t code = 0;
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
Poco::Event event;
|
||||
|
||||
auto callback = [&](const Coordination::ListResponse & response)
|
||||
{
|
||||
code = response.error;
|
||||
if (!code)
|
||||
if (code == Coordination::Error::ZOK)
|
||||
{
|
||||
res = response.names;
|
||||
if (stat)
|
||||
@ -251,37 +251,37 @@ Strings ZooKeeper::getChildrenWatch(
|
||||
return res;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::tryGetChildren(const std::string & path, Strings & res,
|
||||
Coordination::Error ZooKeeper::tryGetChildren(const std::string & path, Strings & res,
|
||||
Coordination::Stat * stat, const EventPtr & watch)
|
||||
{
|
||||
int32_t code = getChildrenImpl(path, res, stat, callbackForEvent(watch));
|
||||
Coordination::Error code = getChildrenImpl(path, res, stat, callbackForEvent(watch));
|
||||
|
||||
if (!(code == Coordination::ZOK || code == Coordination::ZNONODE))
|
||||
if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE))
|
||||
throw KeeperException(code, path);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::tryGetChildrenWatch(const std::string & path, Strings & res,
|
||||
Coordination::Error ZooKeeper::tryGetChildrenWatch(const std::string & path, Strings & res,
|
||||
Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
|
||||
{
|
||||
int32_t code = getChildrenImpl(path, res, stat, watch_callback);
|
||||
Coordination::Error code = getChildrenImpl(path, res, stat, watch_callback);
|
||||
|
||||
if (!(code == Coordination::ZOK || code == Coordination::ZNONODE))
|
||||
if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE))
|
||||
throw KeeperException(code, path);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created)
|
||||
Coordination::Error ZooKeeper::createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created)
|
||||
{
|
||||
int32_t code = 0;
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
Poco::Event event;
|
||||
|
||||
auto callback = [&](const Coordination::CreateResponse & response)
|
||||
{
|
||||
code = response.error;
|
||||
if (!code)
|
||||
if (code == Coordination::Error::ZOK)
|
||||
path_created = response.path_created;
|
||||
event.set();
|
||||
};
|
||||
@ -298,20 +298,20 @@ std::string ZooKeeper::create(const std::string & path, const std::string & data
|
||||
return path_created;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::tryCreate(const std::string & path, const std::string & data, int32_t mode, std::string & path_created)
|
||||
Coordination::Error ZooKeeper::tryCreate(const std::string & path, const std::string & data, int32_t mode, std::string & path_created)
|
||||
{
|
||||
int32_t code = createImpl(path, data, mode, path_created);
|
||||
Coordination::Error code = createImpl(path, data, mode, path_created);
|
||||
|
||||
if (!(code == Coordination::ZOK ||
|
||||
code == Coordination::ZNONODE ||
|
||||
code == Coordination::ZNODEEXISTS ||
|
||||
code == Coordination::ZNOCHILDRENFOREPHEMERALS))
|
||||
if (!(code == Coordination::Error::ZOK ||
|
||||
code == Coordination::Error::ZNONODE ||
|
||||
code == Coordination::Error::ZNODEEXISTS ||
|
||||
code == Coordination::Error::ZNOCHILDRENFOREPHEMERALS))
|
||||
throw KeeperException(code, path);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::tryCreate(const std::string & path, const std::string & data, int32_t mode)
|
||||
Coordination::Error ZooKeeper::tryCreate(const std::string & path, const std::string & data, int32_t mode)
|
||||
{
|
||||
std::string path_created;
|
||||
return tryCreate(path, data, mode, path_created);
|
||||
@ -320,9 +320,9 @@ int32_t ZooKeeper::tryCreate(const std::string & path, const std::string & data,
|
||||
void ZooKeeper::createIfNotExists(const std::string & path, const std::string & data)
|
||||
{
|
||||
std::string path_created;
|
||||
int32_t code = createImpl(path, data, CreateMode::Persistent, path_created);
|
||||
Coordination::Error code = createImpl(path, data, CreateMode::Persistent, path_created);
|
||||
|
||||
if (code == Coordination::ZOK || code == Coordination::ZNODEEXISTS)
|
||||
if (code == Coordination::Error::ZOK || code == Coordination::Error::ZNODEEXISTS)
|
||||
return;
|
||||
else
|
||||
throw KeeperException(code, path);
|
||||
@ -341,14 +341,14 @@ void ZooKeeper::createAncestors(const std::string & path)
|
||||
}
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::removeImpl(const std::string & path, int32_t version)
|
||||
Coordination::Error ZooKeeper::removeImpl(const std::string & path, int32_t version)
|
||||
{
|
||||
int32_t code = 0;
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
Poco::Event event;
|
||||
|
||||
auto callback = [&](const Coordination::RemoveResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
code = response.error;
|
||||
event.set();
|
||||
};
|
||||
@ -363,26 +363,26 @@ void ZooKeeper::remove(const std::string & path, int32_t version)
|
||||
check(tryRemove(path, version), path);
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::tryRemove(const std::string & path, int32_t version)
|
||||
Coordination::Error ZooKeeper::tryRemove(const std::string & path, int32_t version)
|
||||
{
|
||||
int32_t code = removeImpl(path, version);
|
||||
if (!(code == Coordination::ZOK ||
|
||||
code == Coordination::ZNONODE ||
|
||||
code == Coordination::ZBADVERSION ||
|
||||
code == Coordination::ZNOTEMPTY))
|
||||
Coordination::Error code = removeImpl(path, version);
|
||||
if (!(code == Coordination::Error::ZOK ||
|
||||
code == Coordination::Error::ZNONODE ||
|
||||
code == Coordination::Error::ZBADVERSION ||
|
||||
code == Coordination::Error::ZNOTEMPTY))
|
||||
throw KeeperException(code, path);
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::existsImpl(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
|
||||
Coordination::Error ZooKeeper::existsImpl(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
|
||||
{
|
||||
int32_t code = 0;
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
Poco::Event event;
|
||||
|
||||
auto callback = [&](const Coordination::ExistsResponse & response)
|
||||
{
|
||||
code = response.error;
|
||||
if (!code && stat)
|
||||
if (code == Coordination::Error::ZOK && stat)
|
||||
*stat = response.stat;
|
||||
event.set();
|
||||
};
|
||||
@ -399,22 +399,22 @@ bool ZooKeeper::exists(const std::string & path, Coordination::Stat * stat, cons
|
||||
|
||||
bool ZooKeeper::existsWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
|
||||
{
|
||||
int32_t code = existsImpl(path, stat, watch_callback);
|
||||
Coordination::Error code = existsImpl(path, stat, watch_callback);
|
||||
|
||||
if (!(code == Coordination::ZOK || code == Coordination::ZNONODE))
|
||||
if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE))
|
||||
throw KeeperException(code, path);
|
||||
return code != Coordination::ZNONODE;
|
||||
return code != Coordination::Error::ZNONODE;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
|
||||
Coordination::Error ZooKeeper::getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
|
||||
{
|
||||
int32_t code = 0;
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
Poco::Event event;
|
||||
|
||||
auto callback = [&](const Coordination::GetResponse & response)
|
||||
{
|
||||
code = response.error;
|
||||
if (!code)
|
||||
if (code == Coordination::Error::ZOK)
|
||||
{
|
||||
res = response.data;
|
||||
if (stat)
|
||||
@ -431,7 +431,7 @@ int32_t ZooKeeper::getImpl(const std::string & path, std::string & res, Coordina
|
||||
|
||||
std::string ZooKeeper::get(const std::string & path, Coordination::Stat * stat, const EventPtr & watch)
|
||||
{
|
||||
int32_t code = 0;
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
std::string res;
|
||||
if (tryGet(path, res, stat, watch, &code))
|
||||
return res;
|
||||
@ -441,7 +441,7 @@ std::string ZooKeeper::get(const std::string & path, Coordination::Stat * stat,
|
||||
|
||||
std::string ZooKeeper::getWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
|
||||
{
|
||||
int32_t code = 0;
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
std::string res;
|
||||
if (tryGetWatch(path, res, stat, watch_callback, &code))
|
||||
return res;
|
||||
@ -449,34 +449,44 @@ std::string ZooKeeper::getWatch(const std::string & path, Coordination::Stat * s
|
||||
throw KeeperException("Can't get data for node " + path + ": node doesn't exist", code);
|
||||
}
|
||||
|
||||
bool ZooKeeper::tryGet(const std::string & path, std::string & res, Coordination::Stat * stat, const EventPtr & watch, int * return_code)
|
||||
bool ZooKeeper::tryGet(
|
||||
const std::string & path,
|
||||
std::string & res,
|
||||
Coordination::Stat * stat,
|
||||
const EventPtr & watch,
|
||||
Coordination::Error * return_code)
|
||||
{
|
||||
return tryGetWatch(path, res, stat, callbackForEvent(watch), return_code);
|
||||
}
|
||||
|
||||
bool ZooKeeper::tryGetWatch(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback, int * return_code)
|
||||
bool ZooKeeper::tryGetWatch(
|
||||
const std::string & path,
|
||||
std::string & res,
|
||||
Coordination::Stat * stat,
|
||||
Coordination::WatchCallback watch_callback,
|
||||
Coordination::Error * return_code)
|
||||
{
|
||||
int32_t code = getImpl(path, res, stat, watch_callback);
|
||||
Coordination::Error code = getImpl(path, res, stat, watch_callback);
|
||||
|
||||
if (!(code == Coordination::ZOK || code == Coordination::ZNONODE))
|
||||
if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE))
|
||||
throw KeeperException(code, path);
|
||||
|
||||
if (return_code)
|
||||
*return_code = code;
|
||||
|
||||
return code == Coordination::ZOK;
|
||||
return code == Coordination::Error::ZOK;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::setImpl(const std::string & path, const std::string & data,
|
||||
Coordination::Error ZooKeeper::setImpl(const std::string & path, const std::string & data,
|
||||
int32_t version, Coordination::Stat * stat)
|
||||
{
|
||||
int32_t code = 0;
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
Poco::Event event;
|
||||
|
||||
auto callback = [&](const Coordination::SetResponse & response)
|
||||
{
|
||||
code = response.error;
|
||||
if (!code && stat)
|
||||
if (code == Coordination::Error::ZOK && stat)
|
||||
*stat = response.stat;
|
||||
event.set();
|
||||
};
|
||||
@ -493,34 +503,34 @@ void ZooKeeper::set(const std::string & path, const std::string & data, int32_t
|
||||
|
||||
void ZooKeeper::createOrUpdate(const std::string & path, const std::string & data, int32_t mode)
|
||||
{
|
||||
int32_t code = trySet(path, data, -1);
|
||||
if (code == Coordination::ZNONODE)
|
||||
Coordination::Error code = trySet(path, data, -1);
|
||||
if (code == Coordination::Error::ZNONODE)
|
||||
{
|
||||
create(path, data, mode);
|
||||
}
|
||||
else if (code != Coordination::ZOK)
|
||||
else if (code != Coordination::Error::ZOK)
|
||||
throw KeeperException(code, path);
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::trySet(const std::string & path, const std::string & data,
|
||||
Coordination::Error ZooKeeper::trySet(const std::string & path, const std::string & data,
|
||||
int32_t version, Coordination::Stat * stat)
|
||||
{
|
||||
int32_t code = setImpl(path, data, version, stat);
|
||||
Coordination::Error code = setImpl(path, data, version, stat);
|
||||
|
||||
if (!(code == Coordination::ZOK ||
|
||||
code == Coordination::ZNONODE ||
|
||||
code == Coordination::ZBADVERSION))
|
||||
if (!(code == Coordination::Error::ZOK ||
|
||||
code == Coordination::Error::ZNONODE ||
|
||||
code == Coordination::Error::ZBADVERSION))
|
||||
throw KeeperException(code, path);
|
||||
return code;
|
||||
}
|
||||
|
||||
|
||||
int32_t ZooKeeper::multiImpl(const Coordination::Requests & requests, Coordination::Responses & responses)
|
||||
Coordination::Error ZooKeeper::multiImpl(const Coordination::Requests & requests, Coordination::Responses & responses)
|
||||
{
|
||||
if (requests.empty())
|
||||
return Coordination::ZOK;
|
||||
return Coordination::Error::ZOK;
|
||||
|
||||
int32_t code = 0;
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
Poco::Event event;
|
||||
|
||||
auto callback = [&](const Coordination::MultiResponse & response)
|
||||
@ -538,15 +548,15 @@ int32_t ZooKeeper::multiImpl(const Coordination::Requests & requests, Coordinati
|
||||
Coordination::Responses ZooKeeper::multi(const Coordination::Requests & requests)
|
||||
{
|
||||
Coordination::Responses responses;
|
||||
int32_t code = multiImpl(requests, responses);
|
||||
Coordination::Error code = multiImpl(requests, responses);
|
||||
KeeperMultiException::check(code, requests, responses);
|
||||
return responses;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::tryMulti(const Coordination::Requests & requests, Coordination::Responses & responses)
|
||||
Coordination::Error ZooKeeper::tryMulti(const Coordination::Requests & requests, Coordination::Responses & responses)
|
||||
{
|
||||
int32_t code = multiImpl(requests, responses);
|
||||
if (code && !Coordination::isUserError(code))
|
||||
Coordination::Error code = multiImpl(requests, responses);
|
||||
if (code != Coordination::Error::ZOK && !Coordination::isUserError(code))
|
||||
throw KeeperException(code);
|
||||
return code;
|
||||
}
|
||||
@ -587,7 +597,7 @@ void ZooKeeper::removeChildrenRecursive(const std::string & path)
|
||||
void ZooKeeper::tryRemoveChildrenRecursive(const std::string & path)
|
||||
{
|
||||
Strings children;
|
||||
if (tryGetChildren(path, children) != Coordination::ZOK)
|
||||
if (tryGetChildren(path, children) != Coordination::Error::ZOK)
|
||||
return;
|
||||
while (!children.empty())
|
||||
{
|
||||
@ -609,7 +619,7 @@ void ZooKeeper::tryRemoveChildrenRecursive(const std::string & path)
|
||||
/// this means someone is concurrently removing these children and we will have
|
||||
/// to remove them one by one.
|
||||
Coordination::Responses responses;
|
||||
if (tryMulti(ops, responses) != Coordination::ZOK)
|
||||
if (tryMulti(ops, responses) != Coordination::Error::ZOK)
|
||||
for (const std::string & child : batch)
|
||||
tryRemove(child);
|
||||
}
|
||||
@ -645,7 +655,7 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition &
|
||||
|
||||
auto callback = [state](const Coordination::ExistsResponse & response)
|
||||
{
|
||||
state->code = response.error;
|
||||
state->code = int32_t(response.error);
|
||||
if (state->code)
|
||||
state->event.set();
|
||||
};
|
||||
@ -654,7 +664,7 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition &
|
||||
{
|
||||
if (!state->code)
|
||||
{
|
||||
state->code = response.error;
|
||||
state->code = int32_t(response.error);
|
||||
if (!state->code)
|
||||
state->event_type = response.type;
|
||||
state->event.set();
|
||||
@ -670,11 +680,11 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition &
|
||||
else if (!state->event.tryWait(1000))
|
||||
continue;
|
||||
|
||||
if (state->code == Coordination::ZNONODE)
|
||||
if (state->code == int32_t(Coordination::Error::ZNONODE))
|
||||
return true;
|
||||
|
||||
if (state->code)
|
||||
throw KeeperException(state->code, path);
|
||||
throw KeeperException(static_cast<Coordination::Error>(state->code.load(std::memory_order_seq_cst)), path);
|
||||
|
||||
if (state->event_type == Coordination::DELETED)
|
||||
return true;
|
||||
@ -688,11 +698,6 @@ ZooKeeperPtr ZooKeeper::startNewSession() const
|
||||
}
|
||||
|
||||
|
||||
std::string ZooKeeper::error2string(int32_t code)
|
||||
{
|
||||
return Coordination::errorMessage(code);
|
||||
}
|
||||
|
||||
bool ZooKeeper::expired()
|
||||
{
|
||||
return impl->isExpired();
|
||||
@ -712,7 +717,7 @@ std::future<Coordination::CreateResponse> ZooKeeper::asyncCreate(const std::stri
|
||||
|
||||
auto callback = [promise, path](const Coordination::CreateResponse & response) mutable
|
||||
{
|
||||
if (response.error)
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error)));
|
||||
else
|
||||
promise->set_value(response);
|
||||
@ -730,7 +735,7 @@ std::future<Coordination::GetResponse> ZooKeeper::asyncGet(const std::string & p
|
||||
|
||||
auto callback = [promise, path](const Coordination::GetResponse & response) mutable
|
||||
{
|
||||
if (response.error)
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error)));
|
||||
else
|
||||
promise->set_value(response);
|
||||
@ -748,7 +753,7 @@ std::future<Coordination::GetResponse> ZooKeeper::asyncTryGet(const std::string
|
||||
|
||||
auto callback = [promise, path](const Coordination::GetResponse & response) mutable
|
||||
{
|
||||
if (response.error && response.error != Coordination::ZNONODE)
|
||||
if (response.error != Coordination::Error::ZOK && response.error != Coordination::Error::ZNONODE)
|
||||
promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error)));
|
||||
else
|
||||
promise->set_value(response);
|
||||
@ -765,7 +770,7 @@ std::future<Coordination::ExistsResponse> ZooKeeper::asyncExists(const std::stri
|
||||
|
||||
auto callback = [promise, path](const Coordination::ExistsResponse & response) mutable
|
||||
{
|
||||
if (response.error && response.error != Coordination::ZNONODE)
|
||||
if (response.error != Coordination::Error::ZOK && response.error != Coordination::Error::ZNONODE)
|
||||
promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error)));
|
||||
else
|
||||
promise->set_value(response);
|
||||
@ -782,7 +787,7 @@ std::future<Coordination::SetResponse> ZooKeeper::asyncSet(const std::string & p
|
||||
|
||||
auto callback = [promise, path](const Coordination::SetResponse & response) mutable
|
||||
{
|
||||
if (response.error)
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error)));
|
||||
else
|
||||
promise->set_value(response);
|
||||
@ -799,7 +804,7 @@ std::future<Coordination::ListResponse> ZooKeeper::asyncGetChildren(const std::s
|
||||
|
||||
auto callback = [promise, path](const Coordination::ListResponse & response) mutable
|
||||
{
|
||||
if (response.error)
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error)));
|
||||
else
|
||||
promise->set_value(response);
|
||||
@ -816,7 +821,7 @@ std::future<Coordination::RemoveResponse> ZooKeeper::asyncRemove(const std::stri
|
||||
|
||||
auto callback = [promise, path](const Coordination::RemoveResponse & response) mutable
|
||||
{
|
||||
if (response.error)
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error)));
|
||||
else
|
||||
promise->set_value(response);
|
||||
@ -833,8 +838,13 @@ std::future<Coordination::RemoveResponse> ZooKeeper::asyncTryRemove(const std::s
|
||||
|
||||
auto callback = [promise, path](const Coordination::RemoveResponse & response) mutable
|
||||
{
|
||||
if (response.error && response.error != Coordination::ZNONODE && response.error != Coordination::ZBADVERSION && response.error != Coordination::ZNOTEMPTY)
|
||||
if (response.error != Coordination::Error::ZOK
|
||||
&& response.error != Coordination::Error::ZNONODE
|
||||
&& response.error != Coordination::Error::ZBADVERSION
|
||||
&& response.error != Coordination::Error::ZNOTEMPTY)
|
||||
{
|
||||
promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error)));
|
||||
}
|
||||
else
|
||||
promise->set_value(response);
|
||||
};
|
||||
@ -864,7 +874,7 @@ std::future<Coordination::MultiResponse> ZooKeeper::asyncMulti(const Coordinatio
|
||||
|
||||
auto callback = [promise](const Coordination::MultiResponse & response) mutable
|
||||
{
|
||||
if (response.error)
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
promise->set_exception(std::make_exception_ptr(KeeperException(response.error)));
|
||||
else
|
||||
promise->set_value(response);
|
||||
@ -874,7 +884,7 @@ std::future<Coordination::MultiResponse> ZooKeeper::asyncMulti(const Coordinatio
|
||||
return future;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::tryMultiNoThrow(const Coordination::Requests & requests, Coordination::Responses & responses)
|
||||
Coordination::Error ZooKeeper::tryMultiNoThrow(const Coordination::Requests & requests, Coordination::Responses & responses)
|
||||
{
|
||||
try
|
||||
{
|
||||
@ -887,24 +897,24 @@ int32_t ZooKeeper::tryMultiNoThrow(const Coordination::Requests & requests, Coor
|
||||
}
|
||||
|
||||
|
||||
size_t KeeperMultiException::getFailedOpIndex(int32_t exception_code, const Coordination::Responses & responses)
|
||||
size_t KeeperMultiException::getFailedOpIndex(Coordination::Error exception_code, const Coordination::Responses & responses)
|
||||
{
|
||||
if (responses.empty())
|
||||
throw DB::Exception("Responses for multi transaction is empty", DB::ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
for (size_t index = 0, size = responses.size(); index < size; ++index)
|
||||
if (responses[index]->error)
|
||||
if (responses[index]->error != Coordination::Error::ZOK)
|
||||
return index;
|
||||
|
||||
if (!Coordination::isUserError(exception_code))
|
||||
throw DB::Exception("There are no failed OPs because '" + ZooKeeper::error2string(exception_code) + "' is not valid response code for that",
|
||||
throw DB::Exception("There are no failed OPs because '" + std::string(Coordination::errorMessage(exception_code)) + "' is not valid response code for that",
|
||||
DB::ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
throw DB::Exception("There is no failed OpResult", DB::ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
|
||||
KeeperMultiException::KeeperMultiException(int32_t exception_code, const Coordination::Requests & requests_, const Coordination::Responses & responses_)
|
||||
KeeperMultiException::KeeperMultiException(Coordination::Error exception_code, const Coordination::Requests & requests_, const Coordination::Responses & responses_)
|
||||
: KeeperException("Transaction failed", exception_code),
|
||||
requests(requests_), responses(responses_), failed_op_index(getFailedOpIndex(exception_code, responses))
|
||||
{
|
||||
@ -917,9 +927,10 @@ std::string KeeperMultiException::getPathForFirstFailedOp() const
|
||||
return requests[failed_op_index]->getPath();
|
||||
}
|
||||
|
||||
void KeeperMultiException::check(int32_t exception_code, const Coordination::Requests & requests, const Coordination::Responses & responses)
|
||||
void KeeperMultiException::check(
|
||||
Coordination::Error exception_code, const Coordination::Requests & requests, const Coordination::Responses & responses)
|
||||
{
|
||||
if (!exception_code)
|
||||
if (exception_code == Coordination::Error::ZOK)
|
||||
return;
|
||||
|
||||
if (Coordination::isUserError(exception_code))
|
||||
|
||||
@ -99,8 +99,8 @@ public:
|
||||
/// * The parent is ephemeral.
|
||||
/// * The node already exists.
|
||||
/// In case of other errors throws an exception.
|
||||
int32_t tryCreate(const std::string & path, const std::string & data, int32_t mode, std::string & path_created);
|
||||
int32_t tryCreate(const std::string & path, const std::string & data, int32_t mode);
|
||||
Coordination::Error tryCreate(const std::string & path, const std::string & data, int32_t mode, std::string & path_created);
|
||||
Coordination::Error tryCreate(const std::string & path, const std::string & data, int32_t mode);
|
||||
|
||||
/// Create a Persistent node.
|
||||
/// Does nothing if the node already exists.
|
||||
@ -117,7 +117,7 @@ public:
|
||||
/// * The node doesn't exist
|
||||
/// * Versions don't match
|
||||
/// * The node has children.
|
||||
int32_t tryRemove(const std::string & path, int32_t version = -1);
|
||||
Coordination::Error tryRemove(const std::string & path, int32_t version = -1);
|
||||
|
||||
bool exists(const std::string & path, Coordination::Stat * stat = nullptr, const EventPtr & watch = nullptr);
|
||||
bool existsWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback);
|
||||
@ -127,9 +127,11 @@ public:
|
||||
|
||||
/// Doesn't not throw in the following cases:
|
||||
/// * The node doesn't exist. Returns false in this case.
|
||||
bool tryGet(const std::string & path, std::string & res, Coordination::Stat * stat = nullptr, const EventPtr & watch = nullptr, int * code = nullptr);
|
||||
bool tryGet(const std::string & path, std::string & res, Coordination::Stat * stat = nullptr, const EventPtr & watch = nullptr,
|
||||
Coordination::Error * code = nullptr);
|
||||
|
||||
bool tryGetWatch(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback, int * code = nullptr);
|
||||
bool tryGetWatch(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback,
|
||||
Coordination::Error * code = nullptr);
|
||||
|
||||
void set(const std::string & path, const std::string & data,
|
||||
int32_t version = -1, Coordination::Stat * stat = nullptr);
|
||||
@ -140,7 +142,7 @@ public:
|
||||
/// Doesn't not throw in the following cases:
|
||||
/// * The node doesn't exist.
|
||||
/// * Versions do not match.
|
||||
int32_t trySet(const std::string & path, const std::string & data,
|
||||
Coordination::Error trySet(const std::string & path, const std::string & data,
|
||||
int32_t version = -1, Coordination::Stat * stat = nullptr);
|
||||
|
||||
Strings getChildren(const std::string & path,
|
||||
@ -153,11 +155,11 @@ public:
|
||||
|
||||
/// Doesn't not throw in the following cases:
|
||||
/// * The node doesn't exist.
|
||||
int32_t tryGetChildren(const std::string & path, Strings & res,
|
||||
Coordination::Error tryGetChildren(const std::string & path, Strings & res,
|
||||
Coordination::Stat * stat = nullptr,
|
||||
const EventPtr & watch = nullptr);
|
||||
|
||||
int32_t tryGetChildrenWatch(const std::string & path, Strings & res,
|
||||
Coordination::Error tryGetChildrenWatch(const std::string & path, Strings & res,
|
||||
Coordination::Stat * stat,
|
||||
Coordination::WatchCallback watch_callback);
|
||||
|
||||
@ -166,9 +168,9 @@ public:
|
||||
Coordination::Responses multi(const Coordination::Requests & requests);
|
||||
/// Throws only if some operation has returned an "unexpected" error
|
||||
/// - an error that would cause the corresponding try- method to throw.
|
||||
int32_t tryMulti(const Coordination::Requests & requests, Coordination::Responses & responses);
|
||||
Coordination::Error tryMulti(const Coordination::Requests & requests, Coordination::Responses & responses);
|
||||
/// Throws nothing (even session expired errors)
|
||||
int32_t tryMultiNoThrow(const Coordination::Requests & requests, Coordination::Responses & responses);
|
||||
Coordination::Error tryMultiNoThrow(const Coordination::Requests & requests, Coordination::Responses & responses);
|
||||
|
||||
Int64 getClientID();
|
||||
|
||||
@ -238,8 +240,6 @@ public:
|
||||
/// Like the previous one but don't throw any exceptions on future.get()
|
||||
FutureMulti tryAsyncMulti(const Coordination::Requests & ops);
|
||||
|
||||
static std::string error2string(int32_t code);
|
||||
|
||||
private:
|
||||
friend class EphemeralNodeHolder;
|
||||
|
||||
@ -250,13 +250,15 @@ private:
|
||||
void tryRemoveChildrenRecursive(const std::string & path);
|
||||
|
||||
/// The following methods don't throw exceptions but return error codes.
|
||||
int32_t createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created);
|
||||
int32_t removeImpl(const std::string & path, int32_t version);
|
||||
int32_t getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback);
|
||||
int32_t setImpl(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat);
|
||||
int32_t getChildrenImpl(const std::string & path, Strings & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback);
|
||||
int32_t multiImpl(const Coordination::Requests & requests, Coordination::Responses & responses);
|
||||
int32_t existsImpl(const std::string & path, Coordination::Stat * stat_, Coordination::WatchCallback watch_callback);
|
||||
Coordination::Error createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created);
|
||||
Coordination::Error removeImpl(const std::string & path, int32_t version);
|
||||
Coordination::Error getImpl(
|
||||
const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback);
|
||||
Coordination::Error setImpl(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat);
|
||||
Coordination::Error getChildrenImpl(
|
||||
const std::string & path, Strings & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback);
|
||||
Coordination::Error multiImpl(const Coordination::Requests & requests, Coordination::Responses & responses);
|
||||
Coordination::Error existsImpl(const std::string & path, Coordination::Stat * stat_, Coordination::WatchCallback watch_callback);
|
||||
|
||||
std::unique_ptr<Coordination::IKeeper> impl;
|
||||
|
||||
|
||||
@ -52,13 +52,6 @@ namespace CurrentMetrics
|
||||
extern const Metric ZooKeeperWatch;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int SUPPORT_IS_DISABLED;
|
||||
}
|
||||
}
|
||||
|
||||
/** ZooKeeper wire protocol.
|
||||
|
||||
@ -335,6 +328,13 @@ static void read(int32_t & x, ReadBuffer & in)
|
||||
x = __builtin_bswap32(x);
|
||||
}
|
||||
|
||||
static void read(Error & x, ReadBuffer & in)
|
||||
{
|
||||
int32_t code;
|
||||
read(code, in);
|
||||
x = Error(code);
|
||||
}
|
||||
|
||||
static void read(bool & x, ReadBuffer & in)
|
||||
{
|
||||
readBinary(x, in);
|
||||
@ -353,10 +353,10 @@ static void read(String & s, ReadBuffer & in)
|
||||
}
|
||||
|
||||
if (size < 0)
|
||||
throw Exception("Negative size while reading string from ZooKeeper", ZMARSHALLINGERROR);
|
||||
throw Exception("Negative size while reading string from ZooKeeper", Error::ZMARSHALLINGERROR);
|
||||
|
||||
if (size > MAX_STRING_OR_ARRAY_SIZE)
|
||||
throw Exception("Too large string size while reading from ZooKeeper", ZMARSHALLINGERROR);
|
||||
throw Exception("Too large string size while reading from ZooKeeper", Error::ZMARSHALLINGERROR);
|
||||
|
||||
s.resize(size);
|
||||
in.read(s.data(), size);
|
||||
@ -367,7 +367,7 @@ template <size_t N> void read(std::array<char, N> & s, ReadBuffer & in)
|
||||
int32_t size = 0;
|
||||
read(size, in);
|
||||
if (size != N)
|
||||
throw Exception("Unexpected array size while reading from ZooKeeper", ZMARSHALLINGERROR);
|
||||
throw Exception("Unexpected array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR);
|
||||
in.read(s.data(), N);
|
||||
}
|
||||
|
||||
@ -391,9 +391,9 @@ template <typename T> void read(std::vector<T> & arr, ReadBuffer & in)
|
||||
int32_t size = 0;
|
||||
read(size, in);
|
||||
if (size < 0)
|
||||
throw Exception("Negative size while reading array from ZooKeeper", ZMARSHALLINGERROR);
|
||||
throw Exception("Negative size while reading array from ZooKeeper", Error::ZMARSHALLINGERROR);
|
||||
if (size > MAX_STRING_OR_ARRAY_SIZE)
|
||||
throw Exception("Too large array size while reading from ZooKeeper", ZMARSHALLINGERROR);
|
||||
throw Exception("Too large array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR);
|
||||
arr.resize(size);
|
||||
for (auto & elem : arr)
|
||||
read(elem, in);
|
||||
@ -489,7 +489,7 @@ struct ZooKeeperCloseResponse final : ZooKeeperResponse
|
||||
{
|
||||
void readImpl(ReadBuffer &) override
|
||||
{
|
||||
throw Exception("Received response for close request", ZRUNTIMEINCONSISTENCY);
|
||||
throw Exception("Received response for close request", Error::ZRUNTIMEINCONSISTENCY);
|
||||
}
|
||||
};
|
||||
|
||||
@ -650,12 +650,12 @@ struct ZooKeeperErrorResponse final : ErrorResponse, ZooKeeperResponse
|
||||
{
|
||||
void readImpl(ReadBuffer & in) override
|
||||
{
|
||||
int32_t read_error;
|
||||
Coordination::Error read_error;
|
||||
Coordination::read(read_error, in);
|
||||
|
||||
if (read_error != error)
|
||||
throw Exception("Error code in ErrorResponse (" + toString(read_error) + ") doesn't match error code in header (" + toString(error) + ")",
|
||||
ZMARSHALLINGERROR);
|
||||
throw Exception(fmt::format("Error code in ErrorResponse ({}) doesn't match error code in header ({})", read_error, error),
|
||||
Error::ZMARSHALLINGERROR);
|
||||
}
|
||||
};
|
||||
|
||||
@ -691,7 +691,7 @@ struct ZooKeeperMultiRequest final : MultiRequest, ZooKeeperRequest
|
||||
requests.push_back(std::make_shared<ZooKeeperCheckRequest>(*concrete_request_check));
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal command as part of multi ZooKeeper request", ZBADARGUMENTS);
|
||||
throw Exception("Illegal command as part of multi ZooKeeper request", Error::ZBADARGUMENTS);
|
||||
}
|
||||
}
|
||||
|
||||
@ -739,14 +739,14 @@ struct ZooKeeperMultiResponse final : MultiResponse, ZooKeeperResponse
|
||||
{
|
||||
ZooKeeper::OpNum op_num;
|
||||
bool done;
|
||||
int32_t op_error;
|
||||
Error op_error;
|
||||
|
||||
Coordination::read(op_num, in);
|
||||
Coordination::read(done, in);
|
||||
Coordination::read(op_error, in);
|
||||
|
||||
if (done)
|
||||
throw Exception("Not enough results received for multi transaction", ZMARSHALLINGERROR);
|
||||
throw Exception("Not enough results received for multi transaction", Error::ZMARSHALLINGERROR);
|
||||
|
||||
/// op_num == -1 is special for multi transaction.
|
||||
/// For unknown reason, error code is duplicated in header and in response body.
|
||||
@ -754,18 +754,18 @@ struct ZooKeeperMultiResponse final : MultiResponse, ZooKeeperResponse
|
||||
if (op_num == -1)
|
||||
response = std::make_shared<ZooKeeperErrorResponse>();
|
||||
|
||||
if (op_error)
|
||||
if (op_error != Error::ZOK)
|
||||
{
|
||||
response->error = op_error;
|
||||
|
||||
/// Set error for whole transaction.
|
||||
/// If some operations fail, ZK send global error as zero and then send details about each operation.
|
||||
/// It will set error code for first failed operation and it will set special "runtime inconsistency" code for other operations.
|
||||
if (!error && op_error != ZRUNTIMEINCONSISTENCY)
|
||||
if (error == Error::ZOK && op_error != Error::ZRUNTIMEINCONSISTENCY)
|
||||
error = op_error;
|
||||
}
|
||||
|
||||
if (!op_error || op_num == -1)
|
||||
if (op_error == Error::ZOK || op_num == -1)
|
||||
dynamic_cast<ZooKeeperResponse &>(*response).readImpl(in);
|
||||
}
|
||||
|
||||
@ -780,11 +780,11 @@ struct ZooKeeperMultiResponse final : MultiResponse, ZooKeeperResponse
|
||||
Coordination::read(error_read, in);
|
||||
|
||||
if (!done)
|
||||
throw Exception("Too many results received for multi transaction", ZMARSHALLINGERROR);
|
||||
throw Exception("Too many results received for multi transaction", Error::ZMARSHALLINGERROR);
|
||||
if (op_num != -1)
|
||||
throw Exception("Unexpected op_num received at the end of results for multi transaction", ZMARSHALLINGERROR);
|
||||
throw Exception("Unexpected op_num received at the end of results for multi transaction", Error::ZMARSHALLINGERROR);
|
||||
if (error_read != -1)
|
||||
throw Exception("Unexpected error value received at the end of results for multi transaction", ZMARSHALLINGERROR);
|
||||
throw Exception("Unexpected error value received at the end of results for multi transaction", Error::ZMARSHALLINGERROR);
|
||||
}
|
||||
}
|
||||
};
|
||||
@ -883,7 +883,7 @@ void ZooKeeper::connect(
|
||||
Poco::Timespan connection_timeout)
|
||||
{
|
||||
if (nodes.empty())
|
||||
throw Exception("No nodes passed to ZooKeeper constructor", ZBADARGUMENTS);
|
||||
throw Exception("No nodes passed to ZooKeeper constructor", Error::ZBADARGUMENTS);
|
||||
|
||||
static constexpr size_t num_tries = 3;
|
||||
bool connected = false;
|
||||
@ -901,7 +901,8 @@ void ZooKeeper::connect(
|
||||
#if USE_SSL
|
||||
socket = Poco::Net::SecureStreamSocket();
|
||||
#else
|
||||
throw Exception{"Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED};
|
||||
throw Poco::Exception(
|
||||
"Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support.");
|
||||
#endif
|
||||
}
|
||||
else
|
||||
@ -970,7 +971,7 @@ void ZooKeeper::connect(
|
||||
}
|
||||
|
||||
message << fail_reasons.str() << "\n";
|
||||
throw Exception(message.str(), ZCONNECTIONLOSS);
|
||||
throw Exception(message.str(), Error::ZCONNECTIONLOSS);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1005,11 +1006,11 @@ void ZooKeeper::receiveHandshake()
|
||||
|
||||
read(handshake_length);
|
||||
if (handshake_length != 36)
|
||||
throw Exception("Unexpected handshake length received: " + toString(handshake_length), ZMARSHALLINGERROR);
|
||||
throw Exception("Unexpected handshake length received: " + toString(handshake_length), Error::ZMARSHALLINGERROR);
|
||||
|
||||
read(protocol_version_read);
|
||||
if (protocol_version_read != protocol_version)
|
||||
throw Exception("Unexpected protocol version: " + toString(protocol_version_read), ZMARSHALLINGERROR);
|
||||
throw Exception("Unexpected protocol version: " + toString(protocol_version_read), Error::ZMARSHALLINGERROR);
|
||||
|
||||
read(timeout);
|
||||
if (timeout != session_timeout.totalMilliseconds())
|
||||
@ -1032,7 +1033,7 @@ void ZooKeeper::sendAuth(const String & scheme, const String & data)
|
||||
int32_t length;
|
||||
XID read_xid;
|
||||
int64_t zxid;
|
||||
int32_t err;
|
||||
Error err;
|
||||
|
||||
read(length);
|
||||
size_t count_before_event = in->count();
|
||||
@ -1042,16 +1043,16 @@ void ZooKeeper::sendAuth(const String & scheme, const String & data)
|
||||
|
||||
if (read_xid != auth_xid)
|
||||
throw Exception("Unexpected event received in reply to auth request: " + toString(read_xid),
|
||||
ZMARSHALLINGERROR);
|
||||
Error::ZMARSHALLINGERROR);
|
||||
|
||||
int32_t actual_length = in->count() - count_before_event;
|
||||
if (length != actual_length)
|
||||
throw Exception("Response length doesn't match. Expected: " + toString(length) + ", actual: " + toString(actual_length),
|
||||
ZMARSHALLINGERROR);
|
||||
Error::ZMARSHALLINGERROR);
|
||||
|
||||
if (err)
|
||||
throw Exception("Error received in reply to auth request. Code: " + toString(err) + ". Message: " + String(errorMessage(err)),
|
||||
ZMARSHALLINGERROR);
|
||||
if (err != Error::ZOK)
|
||||
throw Exception("Error received in reply to auth request. Code: " + toString(int32_t(err)) + ". Message: " + String(errorMessage(err)),
|
||||
Error::ZMARSHALLINGERROR);
|
||||
}
|
||||
|
||||
|
||||
@ -1154,7 +1155,7 @@ void ZooKeeper::receiveThread()
|
||||
earliest_operation = operations.begin()->second;
|
||||
auto earliest_operation_deadline = earliest_operation->time + std::chrono::microseconds(operation_timeout.totalMicroseconds());
|
||||
if (now > earliest_operation_deadline)
|
||||
throw Exception("Operation timeout (deadline already expired) for path: " + earliest_operation->request->getPath(), ZOPERATIONTIMEOUT);
|
||||
throw Exception("Operation timeout (deadline already expired) for path: " + earliest_operation->request->getPath(), Error::ZOPERATIONTIMEOUT);
|
||||
max_wait = std::chrono::duration_cast<std::chrono::microseconds>(earliest_operation_deadline - now).count();
|
||||
}
|
||||
}
|
||||
@ -1170,10 +1171,10 @@ void ZooKeeper::receiveThread()
|
||||
else
|
||||
{
|
||||
if (earliest_operation)
|
||||
throw Exception("Operation timeout (no response) for path: " + earliest_operation->request->getPath(), ZOPERATIONTIMEOUT);
|
||||
throw Exception("Operation timeout (no response) for path: " + earliest_operation->request->getPath(), Error::ZOPERATIONTIMEOUT);
|
||||
waited += max_wait;
|
||||
if (waited >= session_timeout.totalMicroseconds())
|
||||
throw Exception("Nothing is received in session timeout", ZOPERATIONTIMEOUT);
|
||||
throw Exception("Nothing is received in session timeout", Error::ZOPERATIONTIMEOUT);
|
||||
|
||||
}
|
||||
|
||||
@ -1193,7 +1194,7 @@ void ZooKeeper::receiveEvent()
|
||||
int32_t length;
|
||||
XID xid;
|
||||
int64_t zxid;
|
||||
int32_t err;
|
||||
Error err;
|
||||
|
||||
read(length);
|
||||
size_t count_before_event = in->count();
|
||||
@ -1206,8 +1207,8 @@ void ZooKeeper::receiveEvent()
|
||||
|
||||
if (xid == ping_xid)
|
||||
{
|
||||
if (err)
|
||||
throw Exception("Received error in heartbeat response: " + String(errorMessage(err)), ZRUNTIMEINCONSISTENCY);
|
||||
if (err != Error::ZOK)
|
||||
throw Exception("Received error in heartbeat response: " + String(errorMessage(err)), Error::ZRUNTIMEINCONSISTENCY);
|
||||
|
||||
response = std::make_shared<ZooKeeperHeartbeatResponse>();
|
||||
}
|
||||
@ -1252,7 +1253,7 @@ void ZooKeeper::receiveEvent()
|
||||
|
||||
auto it = operations.find(xid);
|
||||
if (it == operations.end())
|
||||
throw Exception("Received response for unknown xid", ZRUNTIMEINCONSISTENCY);
|
||||
throw Exception("Received response for unknown xid", Error::ZRUNTIMEINCONSISTENCY);
|
||||
|
||||
/// After this point, we must invoke callback, that we've grabbed from 'operations'.
|
||||
/// Invariant: all callbacks are invoked either in case of success or in case of error.
|
||||
@ -1272,7 +1273,7 @@ void ZooKeeper::receiveEvent()
|
||||
if (!response)
|
||||
response = request_info.request->makeResponse();
|
||||
|
||||
if (err)
|
||||
if (err != Error::ZOK)
|
||||
response->error = err;
|
||||
else
|
||||
{
|
||||
@ -1282,7 +1283,7 @@ void ZooKeeper::receiveEvent()
|
||||
|
||||
int32_t actual_length = in->count() - count_before_event;
|
||||
if (length != actual_length)
|
||||
throw Exception("Response length doesn't match. Expected: " + toString(length) + ", actual: " + toString(actual_length), ZMARSHALLINGERROR);
|
||||
throw Exception("Response length doesn't match. Expected: " + toString(length) + ", actual: " + toString(actual_length), Error::ZMARSHALLINGERROR);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -1294,7 +1295,7 @@ void ZooKeeper::receiveEvent()
|
||||
|
||||
/// In case we cannot read the response, we should indicate it as the error of that type
|
||||
/// when the user cannot assume whether the request was processed or not.
|
||||
response->error = ZCONNECTIONLOSS;
|
||||
response->error = Error::ZCONNECTIONLOSS;
|
||||
if (request_info.callback)
|
||||
request_info.callback(*response);
|
||||
|
||||
@ -1361,8 +1362,8 @@ void ZooKeeper::finalize(bool error_send, bool error_receive)
|
||||
ResponsePtr response = request_info.request->makeResponse();
|
||||
|
||||
response->error = request_info.request->probably_sent
|
||||
? ZCONNECTIONLOSS
|
||||
: ZSESSIONEXPIRED;
|
||||
? Error::ZCONNECTIONLOSS
|
||||
: Error::ZSESSIONEXPIRED;
|
||||
|
||||
if (request_info.callback)
|
||||
{
|
||||
@ -1390,7 +1391,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive)
|
||||
WatchResponse response;
|
||||
response.type = SESSION;
|
||||
response.state = EXPIRED_SESSION;
|
||||
response.error = ZSESSIONEXPIRED;
|
||||
response.error = Error::ZSESSIONEXPIRED;
|
||||
|
||||
for (auto & callback : path_watches.second)
|
||||
{
|
||||
@ -1421,7 +1422,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive)
|
||||
ResponsePtr response = info.request->makeResponse();
|
||||
if (response)
|
||||
{
|
||||
response->error = ZSESSIONEXPIRED;
|
||||
response->error = Error::ZSESSIONEXPIRED;
|
||||
try
|
||||
{
|
||||
info.callback(*response);
|
||||
@ -1437,7 +1438,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive)
|
||||
WatchResponse response;
|
||||
response.type = SESSION;
|
||||
response.state = EXPIRED_SESSION;
|
||||
response.error = ZSESSIONEXPIRED;
|
||||
response.error = Error::ZSESSIONEXPIRED;
|
||||
try
|
||||
{
|
||||
info.watch(response);
|
||||
@ -1466,9 +1467,9 @@ void ZooKeeper::pushRequest(RequestInfo && info)
|
||||
{
|
||||
info.request->xid = next_xid.fetch_add(1);
|
||||
if (info.request->xid == close_xid)
|
||||
throw Exception("xid equal to close_xid", ZSESSIONEXPIRED);
|
||||
throw Exception("xid equal to close_xid", Error::ZSESSIONEXPIRED);
|
||||
if (info.request->xid < 0)
|
||||
throw Exception("XID overflow", ZSESSIONEXPIRED);
|
||||
throw Exception("XID overflow", Error::ZSESSIONEXPIRED);
|
||||
}
|
||||
|
||||
/// We must serialize 'pushRequest' and 'finalize' (from sendThread, receiveThread) calls
|
||||
@ -1478,10 +1479,10 @@ void ZooKeeper::pushRequest(RequestInfo && info)
|
||||
std::lock_guard lock(push_request_mutex);
|
||||
|
||||
if (expired)
|
||||
throw Exception("Session expired", ZSESSIONEXPIRED);
|
||||
throw Exception("Session expired", Error::ZSESSIONEXPIRED);
|
||||
|
||||
if (!requests_queue.tryPush(std::move(info), operation_timeout.totalMilliseconds()))
|
||||
throw Exception("Cannot push request to queue within operation timeout", ZOPERATIONTIMEOUT);
|
||||
throw Exception("Cannot push request to queue within operation timeout", Error::ZOPERATIONTIMEOUT);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -1651,7 +1652,7 @@ void ZooKeeper::close()
|
||||
request_info.request = std::make_shared<ZooKeeperCloseRequest>(std::move(request));
|
||||
|
||||
if (!requests_queue.tryPush(std::move(request_info), operation_timeout.totalMilliseconds()))
|
||||
throw Exception("Cannot push close request to queue within operation timeout", ZOPERATIONTIMEOUT);
|
||||
throw Exception("Cannot push close request to queue within operation timeout", Error::ZOPERATIONTIMEOUT);
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::ZooKeeperClose);
|
||||
}
|
||||
|
||||
@ -86,7 +86,7 @@ TEST(zkutil, MultiAsync)
|
||||
ops.clear();
|
||||
|
||||
auto res = fut.get();
|
||||
ASSERT_EQ(res.error, Coordination::ZOK);
|
||||
ASSERT_EQ(res.error, Coordination::Error::ZOK);
|
||||
ASSERT_EQ(res.responses.size(), 2);
|
||||
}
|
||||
|
||||
@ -126,15 +126,15 @@ TEST(zkutil, MultiAsync)
|
||||
|
||||
/// The test is quite heavy. It is normal if session is expired during this test.
|
||||
/// If we don't check that, the test will be flacky.
|
||||
if (res.error != Coordination::ZSESSIONEXPIRED && res.error != Coordination::ZCONNECTIONLOSS)
|
||||
if (res.error != Coordination::Error::ZSESSIONEXPIRED && res.error != Coordination::Error::ZCONNECTIONLOSS)
|
||||
{
|
||||
ASSERT_EQ(res.error, Coordination::ZNODEEXISTS);
|
||||
ASSERT_EQ(res.error, Coordination::Error::ZNODEEXISTS);
|
||||
ASSERT_EQ(res.responses.size(), 2);
|
||||
}
|
||||
}
|
||||
catch (const Coordination::Exception & e)
|
||||
{
|
||||
if (e.code != Coordination::ZSESSIONEXPIRED && e.code != Coordination::ZCONNECTIONLOSS)
|
||||
if (e.code != Coordination::Error::ZSESSIONEXPIRED && e.code != Coordination::Error::ZCONNECTIONLOSS)
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
@ -39,12 +39,12 @@ int main(int argc, char ** argv)
|
||||
ops.emplace_back(zkutil::makeRemoveRequest("/test/zk_expiration_test", -1));
|
||||
|
||||
Coordination::Responses responses;
|
||||
int32_t code = zk.tryMultiNoThrow(ops, responses);
|
||||
Coordination::Error code = zk.tryMultiNoThrow(ops, responses);
|
||||
|
||||
std::cout << time(nullptr) - time0 << "s: " << zkutil::ZooKeeper::error2string(code) << std::endl;
|
||||
std::cout << time(nullptr) - time0 << "s: " << Coordination::errorMessage(code) << std::endl;
|
||||
try
|
||||
{
|
||||
if (code)
|
||||
if (code != Coordination::Error::ZOK)
|
||||
std::cout << "Path: " << zkutil::KeeperMultiException(code, ops, responses).getPathForFirstFailedOp() << std::endl;
|
||||
}
|
||||
catch (...)
|
||||
|
||||
@ -49,8 +49,8 @@ try
|
||||
zk.create("/test", "old", false, false, {},
|
||||
[&](const CreateResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Error (create) " << response.error << ": " << errorMessage(response.error) << '\n';
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Error (create): " << errorMessage(response.error) << '\n';
|
||||
else
|
||||
std::cerr << "Created path: " << response.path_created << '\n';
|
||||
|
||||
@ -64,8 +64,8 @@ try
|
||||
zk.get("/test",
|
||||
[&](const GetResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Error (get) " << response.error << ": " << errorMessage(response.error) << '\n';
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Error (get): " << errorMessage(response.error) << '\n';
|
||||
else
|
||||
std::cerr << "Value: " << response.data << '\n';
|
||||
|
||||
@ -73,8 +73,8 @@ try
|
||||
},
|
||||
[](const WatchResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Watch (get) on /test, Error " << response.error << ": " << errorMessage(response.error) << '\n';
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Watch (get) on /test, Error: " << errorMessage(response.error) << '\n';
|
||||
else
|
||||
std::cerr << "Watch (get) on /test, path: " << response.path << ", type: " << response.type << '\n';
|
||||
});
|
||||
@ -86,8 +86,8 @@ try
|
||||
zk.set("/test", "new", -1,
|
||||
[&](const SetResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Error (set) " << response.error << ": " << errorMessage(response.error) << '\n';
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Error (set): " << errorMessage(response.error) << '\n';
|
||||
else
|
||||
std::cerr << "Set\n";
|
||||
|
||||
@ -101,8 +101,8 @@ try
|
||||
zk.list("/",
|
||||
[&](const ListResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Error (list) " << response.error << ": " << errorMessage(response.error) << '\n';
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Error (list): " << errorMessage(response.error) << '\n';
|
||||
else
|
||||
{
|
||||
std::cerr << "Children:\n";
|
||||
@ -114,8 +114,8 @@ try
|
||||
},
|
||||
[](const WatchResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Watch (list) on /, Error " << response.error << ": " << errorMessage(response.error) << '\n';
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Watch (list) on /, Error: " << errorMessage(response.error) << '\n';
|
||||
else
|
||||
std::cerr << "Watch (list) on /, path: " << response.path << ", type: " << response.type << '\n';
|
||||
});
|
||||
@ -127,8 +127,8 @@ try
|
||||
zk.exists("/test",
|
||||
[&](const ExistsResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Error (exists) " << response.error << ": " << errorMessage(response.error) << '\n';
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Error (exists): " << errorMessage(response.error) << '\n';
|
||||
else
|
||||
std::cerr << "Exists\n";
|
||||
|
||||
@ -136,8 +136,8 @@ try
|
||||
},
|
||||
[](const WatchResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Watch (exists) on /test, Error " << response.error << ": " << errorMessage(response.error) << '\n';
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Watch (exists) on /test, Error: " << errorMessage(response.error) << '\n';
|
||||
else
|
||||
std::cerr << "Watch (exists) on /test, path: " << response.path << ", type: " << response.type << '\n';
|
||||
});
|
||||
@ -148,8 +148,8 @@ try
|
||||
|
||||
zk.remove("/test", -1, [&](const RemoveResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Error (remove) " << response.error << ": " << errorMessage(response.error) << '\n';
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Error (remove): " << errorMessage(response.error) << '\n';
|
||||
else
|
||||
std::cerr << "Removed\n";
|
||||
|
||||
@ -184,13 +184,13 @@ try
|
||||
|
||||
zk.multi(ops, [&](const MultiResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Error (multi) " << response.error << ": " << errorMessage(response.error) << '\n';
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Error (multi): " << errorMessage(response.error) << '\n';
|
||||
else
|
||||
{
|
||||
for (const auto & elem : response.responses)
|
||||
if (elem->error)
|
||||
std::cerr << "Error (elem) " << elem->error << ": " << errorMessage(elem->error) << '\n';
|
||||
if (elem->error != Coordination::Error::ZOK)
|
||||
std::cerr << "Error (elem): " << errorMessage(elem->error) << '\n';
|
||||
|
||||
std::cerr << "Created path: " << dynamic_cast<const CreateResponse &>(*response.responses[0]).path_created << '\n';
|
||||
}
|
||||
|
||||
@ -9,8 +9,8 @@ try
|
||||
|
||||
zookeeper.create("/test", "hello", false, false, {}, [](const Coordination::CreateResponse & response)
|
||||
{
|
||||
if (response.error)
|
||||
std::cerr << "Error " << response.error << ": " << Coordination::errorMessage(response.error) << "\n";
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Error: " << Coordination::errorMessage(response.error) << "\n";
|
||||
else
|
||||
std::cerr << "Path created: " << response.path_created << "\n";
|
||||
});
|
||||
|
||||
18
src/Common/clearPasswordFromCommandLine.cpp
Normal file
18
src/Common/clearPasswordFromCommandLine.cpp
Normal file
@ -0,0 +1,18 @@
|
||||
#include <string.h>
|
||||
#include "clearPasswordFromCommandLine.h"
|
||||
|
||||
void clearPasswordFromCommandLine(int argc, char ** argv)
|
||||
{
|
||||
for (int arg = 1; arg < argc; ++arg)
|
||||
{
|
||||
if (arg + 1 < argc && 0 == strcmp(argv[arg], "--password"))
|
||||
{
|
||||
++arg;
|
||||
memset(argv[arg], 0, strlen(argv[arg]));
|
||||
}
|
||||
else if (0 == strncmp(argv[arg], "--password=", strlen("--password=")))
|
||||
{
|
||||
memset(argv[arg] + strlen("--password="), 0, strlen(argv[arg]) - strlen("--password="));
|
||||
}
|
||||
}
|
||||
}
|
||||
6
src/Common/clearPasswordFromCommandLine.h
Normal file
6
src/Common/clearPasswordFromCommandLine.h
Normal file
@ -0,0 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
/** If there are --password=... or --password ... arguments in command line, replace their values with zero bytes.
|
||||
* This is needed to prevent password exposure in 'ps' and similar tools.
|
||||
*/
|
||||
void clearPasswordFromCommandLine(int argc, char ** argv);
|
||||
@ -30,6 +30,7 @@ SRCS(
|
||||
Config/configReadClient.cpp
|
||||
Config/ConfigReloader.cpp
|
||||
createHardLink.cpp
|
||||
clearPasswordFromCommandLine.cpp
|
||||
CurrentMetrics.cpp
|
||||
CurrentThread.cpp
|
||||
DNSResolver.cpp
|
||||
|
||||
@ -87,7 +87,7 @@
|
||||
#define DBMS_DISTRIBUTED_SIGNATURE_HEADER 0xCAFEDACEull
|
||||
#define DBMS_DISTRIBUTED_SIGNATURE_HEADER_OLD_FORMAT 0xCAFECABEull
|
||||
|
||||
#if !__has_include(<sanitizer/asan_interface.h>)
|
||||
#if !__has_include(<sanitizer/asan_interface.h>) || !defined(ADDRESS_SANITIZER)
|
||||
# define ASAN_UNPOISON_MEMORY_REGION(a, b)
|
||||
# define ASAN_POISON_MEMORY_REGION(a, b)
|
||||
#endif
|
||||
|
||||
@ -360,6 +360,7 @@ struct Settings : public SettingsCollection<Settings>
|
||||
M(SettingBool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \
|
||||
M(SettingUInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \
|
||||
M(SettingBool, optimize_arithmetic_operations_in_aggregate_functions, true, "Move arithmetic operations out of aggregation functions", 0) \
|
||||
M(SettingBool, optimize_duplicate_order_by_and_distinct, true, "Remove duplicate ORDER BY and DISTINCT if it's possible", 0) \
|
||||
M(SettingBool, optimize_if_chain_to_miltiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \
|
||||
M(SettingBool, allow_experimental_alter_materialized_view_structure, false, "Allow atomic alter on Materialized views. Work in progress.", 0) \
|
||||
M(SettingBool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there're constants there", 0) \
|
||||
@ -376,6 +377,7 @@ struct Settings : public SettingsCollection<Settings>
|
||||
M(SettingBool, materialize_ttl_after_modify, true, "Apply TTL for old data, after ALTER MODIFY TTL query", 0) \
|
||||
\
|
||||
M(SettingBool, allow_experimental_geo_types, false, "Allow geo data types such as Point, Ring, Polygon, MultiPolygon", 0) \
|
||||
M(SettingBool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \
|
||||
\
|
||||
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
|
||||
\
|
||||
|
||||
@ -7,6 +7,8 @@
|
||||
*/
|
||||
|
||||
#include <Common/SettingsChanges.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -91,7 +93,16 @@ Field SettingsCollection<Derived>::const_reference::getValue() const
|
||||
template <class Derived>
|
||||
Field SettingsCollection<Derived>::valueToCorrespondingType(size_t index, const Field & value)
|
||||
{
|
||||
try
|
||||
{
|
||||
return members()[index].value_to_corresponding_type(value);
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
e.addMessage(fmt::format("in attempt to set the value of setting to {}",
|
||||
applyVisitor(FieldVisitorToString(), value)));
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -45,6 +45,8 @@ public:
|
||||
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
|
||||
|
||||
bool equals(const IDataType & rhs) const override;
|
||||
|
||||
bool canBePromoted() const override { return false; }
|
||||
};
|
||||
|
||||
/** Tansform-type wrapper for DateTime64, applies given Transform to DateTime64 value or only to a whole part of it.
|
||||
|
||||
@ -16,13 +16,18 @@ void DataTypeUUID::serializeText(const IColumn & column, size_t row_num, WriteBu
|
||||
writeText(UUID(assert_cast<const ColumnUInt128 &>(column).getData()[row_num]), ostr);
|
||||
}
|
||||
|
||||
void DataTypeUUID::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
|
||||
void DataTypeUUID::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
|
||||
{
|
||||
UUID x;
|
||||
readText(x, istr);
|
||||
assert_cast<ColumnUInt128 &>(column).getData().push_back(x);
|
||||
}
|
||||
|
||||
void DataTypeUUID::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
deserializeText(column, istr, settings);
|
||||
}
|
||||
|
||||
void DataTypeUUID::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
|
||||
@ -17,6 +17,7 @@ public:
|
||||
bool equals(const IDataType & rhs) const override;
|
||||
|
||||
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
@ -30,6 +31,8 @@ public:
|
||||
|
||||
bool canBeUsedInBitOperations() const override { return true; }
|
||||
bool canBeInsideNullable() const override { return true; }
|
||||
|
||||
bool canBePromoted() const override { return false; }
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -14,11 +14,6 @@ static bool operator==(const IDataType & left, const IDataType & right)
|
||||
return left.equals(right);
|
||||
}
|
||||
|
||||
std::ostream & operator<<(std::ostream & ostr, const IDataType & dt)
|
||||
{
|
||||
return ostr << dt.getName();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
using namespace DB;
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
#include <DataTypes/DataTypeEnum.h>
|
||||
#include <DataTypes/getLeastSupertype.h>
|
||||
|
||||
#include <Interpreters/convertFieldToType.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
|
||||
#include <Functions/IFunctionAdaptors.h>
|
||||
@ -51,7 +52,6 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int TOO_LARGE_STRING_SIZE;
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int LOGICAL_ERROR;
|
||||
@ -812,94 +812,51 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
bool executeDateOrDateTimeOrEnumOrUUIDWithConstString(
|
||||
bool executeWithConstString(
|
||||
Block & block, size_t result, const IColumn * col_left_untyped, const IColumn * col_right_untyped,
|
||||
const DataTypePtr & left_type, const DataTypePtr & right_type, bool left_is_num, size_t input_rows_count)
|
||||
const DataTypePtr & left_type, const DataTypePtr & right_type, size_t input_rows_count)
|
||||
{
|
||||
/// This is no longer very special case - comparing dates, datetimes, and enumerations with a string constant.
|
||||
const IColumn * column_string_untyped = !left_is_num ? col_left_untyped : col_right_untyped;
|
||||
const IColumn * column_number = left_is_num ? col_left_untyped : col_right_untyped;
|
||||
const IDataType * number_type = left_is_num ? left_type.get() : right_type.get();
|
||||
/// To compare something with const string, we cast constant to appropriate type and compare as usual.
|
||||
/// It is ok to throw exception if value is not convertible.
|
||||
/// We should deal with possible overflows, e.g. toUInt8(1) = '257' should return false.
|
||||
|
||||
WhichDataType which(number_type);
|
||||
const ColumnConst * left_const = checkAndGetColumnConstStringOrFixedString(col_left_untyped);
|
||||
const ColumnConst * right_const = checkAndGetColumnConstStringOrFixedString(col_right_untyped);
|
||||
|
||||
const bool legal_types = which.isDateOrDateTime() || which.isEnum() || which.isUUID();
|
||||
|
||||
const auto column_string = checkAndGetColumnConst<ColumnString>(column_string_untyped);
|
||||
if (!column_string || !legal_types)
|
||||
if (!left_const && !right_const)
|
||||
return false;
|
||||
|
||||
StringRef string_value = column_string->getDataAt(0);
|
||||
const IDataType * type_string = left_const ? left_type.get() : right_type.get();
|
||||
const DataTypePtr & type_to_compare = !left_const ? left_type : right_type;
|
||||
|
||||
if (which.isDate())
|
||||
Field string_value = left_const ? left_const->getField() : right_const->getField();
|
||||
Field converted = convertFieldToType(string_value, *type_to_compare, type_string);
|
||||
|
||||
/// If not possible to convert, comparison with =, <, >, <=, >= yields to false and comparison with != yields to true.
|
||||
if (converted.isNull())
|
||||
{
|
||||
DayNum date;
|
||||
ReadBufferFromMemory in(string_value.data, string_value.size);
|
||||
readDateText(date, in);
|
||||
if (!in.eof())
|
||||
throw Exception("String is too long for Date: " + string_value.toString(), ErrorCodes::TOO_LARGE_STRING_SIZE);
|
||||
|
||||
ColumnPtr parsed_const_date_holder = DataTypeDate().createColumnConst(input_rows_count, date);
|
||||
const ColumnConst * parsed_const_date = assert_cast<const ColumnConst *>(parsed_const_date_holder.get());
|
||||
executeNumLeftType<DataTypeDate::FieldType>(block, result,
|
||||
left_is_num ? col_left_untyped : parsed_const_date,
|
||||
left_is_num ? parsed_const_date : col_right_untyped);
|
||||
block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count,
|
||||
std::is_same_v<Op<int, int>, NotEqualsOp<int, int>>);
|
||||
}
|
||||
else if (which.isDateTime())
|
||||
else
|
||||
{
|
||||
time_t date_time;
|
||||
ReadBufferFromMemory in(string_value.data, string_value.size);
|
||||
readDateTimeText(date_time, in, dynamic_cast<const DataTypeDateTime &>(*number_type).getTimeZone());
|
||||
if (!in.eof())
|
||||
throw Exception("String is too long for DateTime: " + string_value.toString(), ErrorCodes::TOO_LARGE_STRING_SIZE);
|
||||
auto column_converted = type_to_compare->createColumnConst(input_rows_count, converted);
|
||||
|
||||
ColumnPtr parsed_const_date_time_holder = DataTypeDateTime().createColumnConst(input_rows_count, UInt64(date_time));
|
||||
const ColumnConst * parsed_const_date_time = assert_cast<const ColumnConst *>(parsed_const_date_time_holder.get());
|
||||
executeNumLeftType<DataTypeDateTime::FieldType>(block, result,
|
||||
left_is_num ? col_left_untyped : parsed_const_date_time,
|
||||
left_is_num ? parsed_const_date_time : col_right_untyped);
|
||||
}
|
||||
else if (which.isUUID())
|
||||
Block tmp_block
|
||||
{
|
||||
UUID uuid;
|
||||
ReadBufferFromMemory in(string_value.data, string_value.size);
|
||||
readText(uuid, in);
|
||||
if (!in.eof())
|
||||
throw Exception("String is too long for UUID: " + string_value.toString(), ErrorCodes::TOO_LARGE_STRING_SIZE);
|
||||
{ left_const ? column_converted : col_left_untyped->getPtr(), type_to_compare, "" },
|
||||
{ !left_const ? column_converted : col_right_untyped->getPtr(), type_to_compare, "" },
|
||||
block.getByPosition(result)
|
||||
};
|
||||
|
||||
ColumnPtr parsed_const_uuid_holder = DataTypeUUID().createColumnConst(input_rows_count, uuid);
|
||||
const ColumnConst * parsed_const_uuid = assert_cast<const ColumnConst *>(parsed_const_uuid_holder.get());
|
||||
executeNumLeftType<DataTypeUUID::FieldType>(block, result,
|
||||
left_is_num ? col_left_untyped : parsed_const_uuid,
|
||||
left_is_num ? parsed_const_uuid : col_right_untyped);
|
||||
executeImpl(tmp_block, {0, 1}, 2, input_rows_count);
|
||||
|
||||
block.getByPosition(result).column = std::move(tmp_block.getByPosition(2).column);
|
||||
}
|
||||
|
||||
else if (which.isEnum8())
|
||||
executeEnumWithConstString<DataTypeEnum8>(block, result, column_number, column_string,
|
||||
number_type, left_is_num, input_rows_count);
|
||||
else if (which.isEnum16())
|
||||
executeEnumWithConstString<DataTypeEnum16>(block, result, column_number, column_string,
|
||||
number_type, left_is_num, input_rows_count);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Comparison between DataTypeEnum<T> and string constant containing the name of an enum element
|
||||
template <typename EnumType>
|
||||
void executeEnumWithConstString(
|
||||
Block & block, const size_t result, const IColumn * column_number, const ColumnConst * column_string,
|
||||
const IDataType * type_untyped, const bool left_is_num, size_t input_rows_count)
|
||||
{
|
||||
const auto type = static_cast<const EnumType *>(type_untyped);
|
||||
|
||||
const Field x = castToNearestFieldType(type->getValue(column_string->getValue<String>()));
|
||||
const auto enum_col = type->createColumnConst(input_rows_count, x);
|
||||
|
||||
executeNumLeftType<typename EnumType::FieldType>(block, result,
|
||||
left_is_num ? column_number : enum_col.get(),
|
||||
left_is_num ? enum_col.get() : column_number);
|
||||
}
|
||||
|
||||
void executeTuple(Block & block, size_t result, const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
@ -1124,17 +1081,11 @@ public:
|
||||
bool has_date = left.isDate() || right.isDate();
|
||||
|
||||
if (!((both_represented_by_number && !has_date) /// Do not allow compare date and number.
|
||||
|| (left.isStringOrFixedString() && right.isStringOrFixedString())
|
||||
|| (left.isStringOrFixedString() || right.isStringOrFixedString()) /// Everything can be compared with string by conversion.
|
||||
/// You can compare the date, datetime, or datatime64 and an enumeration with a constant string.
|
||||
|| (left.isString() && right.isDateOrDateTime())
|
||||
|| (left.isDateOrDateTime() && right.isString())
|
||||
|| (left.isDateOrDateTime() && right.isDateOrDateTime() && left.idx == right.idx) /// only date vs date, or datetime vs datetime
|
||||
|| (left.isUUID() && right.isUUID())
|
||||
|| (left.isUUID() && right.isString())
|
||||
|| (left.isString() && right.isUUID())
|
||||
|| (left.isEnum() && right.isEnum() && arguments[0]->getName() == arguments[1]->getName()) /// only equivalent enum type values can be compared against
|
||||
|| (left.isEnum() && right.isString())
|
||||
|| (left.isString() && right.isEnum())
|
||||
|| (left_tuple && right_tuple && left_tuple->getElements().size() == right_tuple->getElements().size())
|
||||
|| (arguments[0]->equals(*arguments[1]))))
|
||||
{
|
||||
@ -1151,7 +1102,8 @@ public:
|
||||
|
||||
if (left_tuple && right_tuple)
|
||||
{
|
||||
auto adaptor = FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(FunctionComparison<Op, Name>::create(context)));
|
||||
auto adaptor = FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(
|
||||
FunctionComparison<Op, Name>::create(context)));
|
||||
|
||||
size_t size = left_tuple->getElements().size();
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
@ -1201,6 +1153,9 @@ public:
|
||||
const bool left_is_num = col_left_untyped->isNumeric();
|
||||
const bool right_is_num = col_right_untyped->isNumeric();
|
||||
|
||||
const bool left_is_string = isStringOrFixedString(which_left);
|
||||
const bool right_is_string = isStringOrFixedString(which_right);
|
||||
|
||||
bool date_and_datetime = (left_type != right_type) &&
|
||||
which_left.isDateOrDateTime() && which_right.isDateOrDateTime();
|
||||
|
||||
@ -1226,64 +1181,14 @@ public:
|
||||
{
|
||||
executeTuple(block, result, col_with_type_and_name_left, col_with_type_and_name_right, input_rows_count);
|
||||
}
|
||||
else if (which_left.idx != which_right.idx
|
||||
&& (which_left.isDateTime64() || which_right.isDateTime64())
|
||||
&& (which_left.isStringOrFixedString() || which_right.isStringOrFixedString()))
|
||||
else if (left_is_string && right_is_string && executeString(block, result, col_left_untyped, col_right_untyped))
|
||||
{
|
||||
/** Special case of comparing DateTime64 against a string.
|
||||
*
|
||||
* Can't be moved to executeDateOrDateTimeOrEnumOrUUIDWithConstString()
|
||||
* since DateTime64 is basically a Decimal, but we do similar things, except type inference.
|
||||
* Outline:
|
||||
* - Extract string content
|
||||
* - Parse it as a ColumnDateTime64 value (same type as DateTime64, means same precision)
|
||||
* - Fabricate a column with type and name
|
||||
* - Compare left and right comlumns as DateTime64 columns.
|
||||
*/
|
||||
|
||||
const size_t datetime64_col_index = which_left.isDateTime64() ? 0 : 1;
|
||||
const size_t string_col_index = which_left.isStringOrFixedString() ? 0 : 1;
|
||||
|
||||
const auto & datetime64_col_with_type_and_name = block.getByPosition(arguments[datetime64_col_index]);
|
||||
const auto & string_col_with_type_and_name = block.getByPosition(arguments[string_col_index]);
|
||||
|
||||
if (!isColumnConst(*string_col_with_type_and_name.column))
|
||||
throw Exception(getName() + ", illegal column type of argument #" + std::to_string(string_col_index)
|
||||
+ " '" + string_col_with_type_and_name.name + "'"
|
||||
" expected const String or const FixedString,"
|
||||
" got " + string_col_with_type_and_name.type->getName(),
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
|
||||
if (datetime64_col_with_type_and_name.column->size() == 0 || string_col_with_type_and_name.column->size() == 0)
|
||||
{
|
||||
// For some reason, when both left and right columns are empty (dry run while building a header block)
|
||||
// executeDecimal() fills result column with bogus value.
|
||||
block.getByPosition(result).column = ColumnUInt8::create();
|
||||
return;
|
||||
}
|
||||
|
||||
auto parsed_tmp_column_holder = datetime64_col_with_type_and_name.type->createColumn();
|
||||
|
||||
else if (executeWithConstString(
|
||||
block, result, col_left_untyped, col_right_untyped,
|
||||
left_type, right_type,
|
||||
input_rows_count))
|
||||
{
|
||||
const StringRef string_value = string_col_with_type_and_name.column->getDataAt(0);
|
||||
ReadBufferFromMemory in(string_value.data, string_value.size);
|
||||
datetime64_col_with_type_and_name.type->deserializeAsWholeText(*parsed_tmp_column_holder, in, FormatSettings{});
|
||||
|
||||
if (!in.eof())
|
||||
throw Exception(getName() + ": String is too long for " + datetime64_col_with_type_and_name.type->getName() + " : " + string_value.toString(), ErrorCodes::TOO_LARGE_STRING_SIZE);
|
||||
}
|
||||
|
||||
// It is necessary to wrap tmp column in ColumnConst to avoid overflow when comparing.
|
||||
// (non-const columns are expected to have same number of rows as every other column in block).
|
||||
const ColumnWithTypeAndName parsed_tmp_col_with_type_and_name{
|
||||
ColumnConst::create(std::move(parsed_tmp_column_holder), 1),
|
||||
datetime64_col_with_type_and_name.type,
|
||||
string_col_with_type_and_name.name};
|
||||
|
||||
executeDecimal(block, result,
|
||||
which_left.isDateTime64() ? datetime64_col_with_type_and_name : parsed_tmp_col_with_type_and_name,
|
||||
which_right.isDateTime64() ? datetime64_col_with_type_and_name : parsed_tmp_col_with_type_and_name);
|
||||
|
||||
}
|
||||
else if (isColumnedAsDecimal(left_type) || isColumnedAsDecimal(right_type))
|
||||
{
|
||||
@ -1294,19 +1199,10 @@ public:
|
||||
|
||||
executeDecimal(block, result, col_with_type_and_name_left, col_with_type_and_name_right);
|
||||
}
|
||||
else if (!left_is_num && !right_is_num && executeString(block, result, col_left_untyped, col_right_untyped))
|
||||
{
|
||||
}
|
||||
else if (left_type->equals(*right_type))
|
||||
{
|
||||
executeGenericIdenticalTypes(block, result, col_left_untyped, col_right_untyped);
|
||||
}
|
||||
else if (executeDateOrDateTimeOrEnumOrUUIDWithConstString(
|
||||
block, result, col_left_untyped, col_right_untyped,
|
||||
left_type, right_type,
|
||||
left_is_num, input_rows_count))
|
||||
{
|
||||
}
|
||||
else
|
||||
{
|
||||
executeGeneric(block, result, col_with_type_and_name_left, col_with_type_and_name_right);
|
||||
|
||||
@ -722,9 +722,9 @@ struct ConvertThroughParsing
|
||||
parsed = ToDataType::tryReadText(vec_to[i], read_buffer, ToDataType::maxPrecision(), vec_to.getScale());
|
||||
else
|
||||
parsed = tryParseImpl<ToDataType>(vec_to[i], read_buffer, local_time_zone);
|
||||
}
|
||||
|
||||
parsed = parsed && isAllRead(read_buffer);
|
||||
}
|
||||
|
||||
if (!parsed)
|
||||
vec_to[i] = 0;
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnNullable.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/FieldVisitorsAccurateComparison.h>
|
||||
#include <Common/memcmpSmall.h>
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
|
||||
@ -1,127 +0,0 @@
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/Regexps.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
|
||||
/** Match all groups of given input string with given re, return array of arrays of matches.
|
||||
*
|
||||
* SELECT extractAllGroups('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)')
|
||||
* should produce:
|
||||
* [['abc', '111'], ['def', '222'], ['ghi', '333']]
|
||||
*/
|
||||
class FunctionExtractAllGroups : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "extractAllGroups";
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionExtractAllGroups>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 2; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return false; }
|
||||
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"haystack", isStringOrFixedString, nullptr, "const String or const FixedString"},
|
||||
{"needle", isStringOrFixedString, isColumnConst, "const String or const FixedString"},
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args);
|
||||
|
||||
/// Two-dimensional array of strings, each `row` of top array represents matching groups.
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()));
|
||||
}
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
|
||||
{
|
||||
const ColumnPtr column_haystack = block.getByPosition(arguments[0]).column;
|
||||
const ColumnPtr column_needle = block.getByPosition(arguments[1]).column;
|
||||
|
||||
const auto needle = typeid_cast<const ColumnConst &>(*column_needle).getValue<String>();
|
||||
|
||||
if (needle.empty())
|
||||
throw Exception(getName() + " length of 'needle' argument must be greater than 0.", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
const auto regexp = Regexps::get<false, false>(needle);
|
||||
const auto & re2 = regexp->getRE2();
|
||||
|
||||
if (!re2)
|
||||
throw Exception("There is no groups in regexp: " + needle, ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
const size_t groups_count = re2->NumberOfCapturingGroups();
|
||||
|
||||
if (!groups_count)
|
||||
throw Exception("There is no groups in regexp: " + needle, ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
// Including 0-group, which is the whole regexp.
|
||||
PODArrayWithStackMemory<re2_st::StringPiece, 128> matched_groups(groups_count + 1);
|
||||
|
||||
ColumnArray::ColumnOffsets::MutablePtr root_offsets_col = ColumnArray::ColumnOffsets::create();
|
||||
ColumnArray::ColumnOffsets::MutablePtr nested_offsets_col = ColumnArray::ColumnOffsets::create();
|
||||
ColumnString::MutablePtr data_col = ColumnString::create();
|
||||
|
||||
auto & root_offsets_data = root_offsets_col->getData();
|
||||
auto & nested_offsets_data = nested_offsets_col->getData();
|
||||
|
||||
root_offsets_data.resize(input_rows_count);
|
||||
ColumnArray::Offset current_root_offset = 0;
|
||||
ColumnArray::Offset current_nested_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
StringRef current_row = column_haystack->getDataAt(i);
|
||||
|
||||
// Extract all non-intersecting matches from haystack except group #0.
|
||||
const auto * pos = current_row.data;
|
||||
const auto * end = pos + current_row.size;
|
||||
while (pos < end
|
||||
&& re2->Match(re2_st::StringPiece(pos, end - pos),
|
||||
0, end - pos, re2_st::RE2::UNANCHORED, matched_groups.data(), matched_groups.size()))
|
||||
{
|
||||
// 1 is to exclude group #0 which is whole re match.
|
||||
for (size_t group = 1; group <= groups_count; ++group)
|
||||
data_col->insertData(matched_groups[group].data(), matched_groups[group].size());
|
||||
|
||||
pos = matched_groups[0].data() + matched_groups[0].size();
|
||||
|
||||
current_nested_offset += groups_count;
|
||||
nested_offsets_data.push_back(current_nested_offset);
|
||||
|
||||
++current_root_offset;
|
||||
}
|
||||
|
||||
root_offsets_data[i] = current_root_offset;
|
||||
}
|
||||
ColumnArray::MutablePtr nested_array_col = ColumnArray::create(std::move(data_col), std::move(nested_offsets_col));
|
||||
ColumnArray::MutablePtr root_array_col = ColumnArray::create(std::move(nested_array_col), std::move(root_offsets_col));
|
||||
block.getByPosition(result).column = std::move(root_array_col);
|
||||
}
|
||||
};
|
||||
|
||||
void registerFunctionExtractAllGroups(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionExtractAllGroups>();
|
||||
}
|
||||
|
||||
}
|
||||
237
src/Functions/extractAllGroups.h
Normal file
237
src/Functions/extractAllGroups.h
Normal file
@ -0,0 +1,237 @@
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionImpl.h>
|
||||
#include <Functions/Regexps.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <Core/iostream_debug_helpers.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
|
||||
enum class ExtractAllGroupsResultKind
|
||||
{
|
||||
VERTICAL,
|
||||
HORIZONTAL
|
||||
};
|
||||
|
||||
|
||||
/** Match all groups of given input string with given re, return array of arrays of matches.
|
||||
*
|
||||
* Depending on `Impl::Kind`, result is either grouped by grop id (Horizontal) or in order of appearance (Vertical):
|
||||
*
|
||||
* SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)')
|
||||
* =>
|
||||
* [['abc', '111'], ['def', '222'], ['ghi', '333']]
|
||||
*
|
||||
* SELECT extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)')
|
||||
* =>
|
||||
* [['abc', 'def', 'ghi'], ['111', '222', '333']
|
||||
*/
|
||||
template <typename Impl>
|
||||
class FunctionExtractAllGroups : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto Kind = Impl::Kind;
|
||||
static constexpr auto name = Impl::Name;
|
||||
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionExtractAllGroups>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 2; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"haystack", isStringOrFixedString, nullptr, "const String or const FixedString"},
|
||||
{"needle", isStringOrFixedString, isColumnConst, "const String or const FixedString"},
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args);
|
||||
|
||||
/// Two-dimensional array of strings, each `row` of top array represents matching groups.
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()));
|
||||
}
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
|
||||
{
|
||||
static const auto MAX_GROUPS_COUNT = 128;
|
||||
|
||||
const ColumnPtr column_haystack = block.getByPosition(arguments[0]).column;
|
||||
const ColumnPtr column_needle = block.getByPosition(arguments[1]).column;
|
||||
|
||||
const auto needle = typeid_cast<const ColumnConst &>(*column_needle).getValue<String>();
|
||||
|
||||
if (needle.empty())
|
||||
throw Exception("Length of 'needle' argument must be greater than 0.", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
using StringPiece = typename Regexps::Regexp::StringPieceType;
|
||||
const auto & regexp = Regexps::get<false, false>(needle)->getRE2();
|
||||
|
||||
if (!regexp)
|
||||
throw Exception("There are no groups in regexp: " + needle, ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
const size_t groups_count = regexp->NumberOfCapturingGroups();
|
||||
|
||||
if (!groups_count)
|
||||
throw Exception("There are no groups in regexp: " + needle, ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if (groups_count > MAX_GROUPS_COUNT - 1)
|
||||
throw Exception("Too many groups in regexp: " + std::to_string(groups_count)
|
||||
+ ", max: " + std::to_string(MAX_GROUPS_COUNT - 1),
|
||||
ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
// Including 0-group, which is the whole regexp.
|
||||
PODArrayWithStackMemory<StringPiece, MAX_GROUPS_COUNT> matched_groups(groups_count + 1);
|
||||
|
||||
ColumnArray::ColumnOffsets::MutablePtr root_offsets_col = ColumnArray::ColumnOffsets::create();
|
||||
ColumnArray::ColumnOffsets::MutablePtr nested_offsets_col = ColumnArray::ColumnOffsets::create();
|
||||
ColumnString::MutablePtr data_col = ColumnString::create();
|
||||
|
||||
auto & root_offsets_data = root_offsets_col->getData();
|
||||
auto & nested_offsets_data = nested_offsets_col->getData();
|
||||
|
||||
ColumnArray::Offset current_root_offset = 0;
|
||||
ColumnArray::Offset current_nested_offset = 0;
|
||||
|
||||
if constexpr (Kind == ExtractAllGroupsResultKind::VERTICAL)
|
||||
{
|
||||
root_offsets_data.resize(input_rows_count);
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
StringRef current_row = column_haystack->getDataAt(i);
|
||||
|
||||
// Extract all non-intersecting matches from haystack except group #0.
|
||||
const auto * pos = current_row.data;
|
||||
const auto * end = pos + current_row.size;
|
||||
while (pos < end
|
||||
&& regexp->Match({pos, static_cast<size_t>(end - pos)},
|
||||
0, end - pos, regexp->UNANCHORED, matched_groups.data(), matched_groups.size()))
|
||||
{
|
||||
// 1 is to exclude group #0 which is whole re match.
|
||||
for (size_t group = 1; group <= groups_count; ++group)
|
||||
data_col->insertData(matched_groups[group].data(), matched_groups[group].size());
|
||||
|
||||
pos = matched_groups[0].data() + matched_groups[0].size();
|
||||
|
||||
current_nested_offset += groups_count;
|
||||
nested_offsets_data.push_back(current_nested_offset);
|
||||
|
||||
++current_root_offset;
|
||||
}
|
||||
|
||||
root_offsets_data[i] = current_root_offset;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
std::vector<StringPiece> all_matches;
|
||||
// number of times RE matched on each row of haystack column.
|
||||
std::vector<size_t> number_of_matches_per_row;
|
||||
|
||||
// we expect RE to match multiple times on each row, `* 8` is arbitrary to reduce number of re-allocations.
|
||||
all_matches.reserve(input_rows_count * groups_count * 8);
|
||||
number_of_matches_per_row.reserve(input_rows_count);
|
||||
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
size_t matches_per_row = 0;
|
||||
|
||||
const auto & current_row = column_haystack->getDataAt(i);
|
||||
|
||||
// Extract all non-intersecting matches from haystack except group #0.
|
||||
const auto * pos = current_row.data;
|
||||
const auto * end = pos + current_row.size;
|
||||
while (pos < end
|
||||
&& regexp->Match({pos, static_cast<size_t>(end - pos)},
|
||||
0, end - pos, regexp->UNANCHORED, matched_groups.data(), matched_groups.size()))
|
||||
{
|
||||
// 1 is to exclude group #0 which is whole re match.
|
||||
for (size_t group = 1; group <= groups_count; ++group)
|
||||
all_matches.push_back(matched_groups[group]);
|
||||
|
||||
pos = matched_groups[0].data() + matched_groups[0].size();
|
||||
|
||||
++matches_per_row;
|
||||
}
|
||||
|
||||
number_of_matches_per_row.push_back(matches_per_row);
|
||||
}
|
||||
|
||||
{
|
||||
size_t total_matched_groups_string_len = 0;
|
||||
for (const auto & m : all_matches)
|
||||
total_matched_groups_string_len += m.length();
|
||||
|
||||
data_col->reserve(total_matched_groups_string_len);
|
||||
}
|
||||
|
||||
nested_offsets_col->reserve(matched_groups.size());
|
||||
root_offsets_col->reserve(groups_count);
|
||||
|
||||
// Re-arrange `all_matches` from:
|
||||
// [
|
||||
// "ROW 0: 1st group 1st match",
|
||||
// "ROW 0: 2nd group 1st match",
|
||||
// ...,
|
||||
// "ROW 0: 1st group 2nd match",
|
||||
// "ROW 0: 2nd group 2nd match",
|
||||
// ...,
|
||||
// "ROW 1: 1st group 1st match",
|
||||
// ...
|
||||
// ]
|
||||
//
|
||||
// into column of 2D arrays:
|
||||
// [
|
||||
// /* all matchig groups from ROW 0 of haystack column */
|
||||
// ["ROW 0: 1st group 1st match", "ROW 0: 1st group 2nd match", ...],
|
||||
// ["ROW 0: 2nd group 1st match", "ROW 0: 2nd group 2nd match", ...],
|
||||
// ...
|
||||
// ],
|
||||
// [
|
||||
// /* all matchig groups from row 1 of haystack column */
|
||||
// ["ROW 1: 1st group 1st match", ...],
|
||||
// ...
|
||||
// ]
|
||||
|
||||
size_t row_offset = 0;
|
||||
for (const auto matches_per_row : number_of_matches_per_row)
|
||||
{
|
||||
const size_t next_row_offset = row_offset + matches_per_row * groups_count;
|
||||
for (size_t group_id = 0; group_id < groups_count; ++group_id)
|
||||
{
|
||||
for (size_t i = row_offset + group_id; i < next_row_offset && i < all_matches.size(); i += groups_count)
|
||||
{
|
||||
const auto & match = all_matches[i];
|
||||
data_col->insertData(match.begin(), match.length());
|
||||
}
|
||||
nested_offsets_col->insertValue(data_col->size());
|
||||
}
|
||||
root_offsets_col->insertValue(nested_offsets_col->size());
|
||||
row_offset = next_row_offset;
|
||||
}
|
||||
}
|
||||
|
||||
ColumnArray::MutablePtr nested_array_col = ColumnArray::create(std::move(data_col), std::move(nested_offsets_col));
|
||||
ColumnArray::MutablePtr root_array_col = ColumnArray::create(std::move(nested_array_col), std::move(root_offsets_col));
|
||||
block.getByPosition(result).column = std::move(root_array_col);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
23
src/Functions/extractAllGroupsHorizontal.cpp
Normal file
23
src/Functions/extractAllGroupsHorizontal.cpp
Normal file
@ -0,0 +1,23 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/extractAllGroups.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
struct HorizontalImpl
|
||||
{
|
||||
static constexpr auto Kind = DB::ExtractAllGroupsResultKind::HORIZONTAL;
|
||||
static constexpr auto Name = "extractAllGroupsHorizontal";
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void registerFunctionExtractAllGroupsHorizontal(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionExtractAllGroups<HorizontalImpl>>();
|
||||
}
|
||||
|
||||
}
|
||||
24
src/Functions/extractAllGroupsVertical.cpp
Normal file
24
src/Functions/extractAllGroupsVertical.cpp
Normal file
@ -0,0 +1,24 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/extractAllGroups.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
struct VerticalImpl
|
||||
{
|
||||
static constexpr auto Kind = DB::ExtractAllGroupsResultKind::VERTICAL;
|
||||
static constexpr auto Name = "extractAllGroupsVertical";
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void registerFunctionExtractAllGroupsVertical(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionExtractAllGroups<VerticalImpl>>();
|
||||
factory.registerAlias("extractAllGroups", VerticalImpl::Name, FunctionFactory::CaseInsensitive);
|
||||
}
|
||||
|
||||
}
|
||||
@ -17,7 +17,6 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
@ -49,7 +48,6 @@ public:
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args);
|
||||
|
||||
/// Two-dimensional array of strings, each `row` of top array represents matching groups.
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
|
||||
}
|
||||
|
||||
@ -61,7 +59,7 @@ public:
|
||||
const auto needle = typeid_cast<const ColumnConst &>(*column_needle).getValue<String>();
|
||||
|
||||
if (needle.empty())
|
||||
throw Exception(getName() + " length of 'needle' argument must be greater than 0.", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
throw Exception(getName() + " length of 'needle' argument must be greater than 0.", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
const auto regexp = Regexps::get<false, false>(needle);
|
||||
const auto & re2 = regexp->getRE2();
|
||||
|
||||
@ -18,7 +18,8 @@ void registerFunctionMultiFuzzyMatchAny(FunctionFactory &);
|
||||
void registerFunctionMultiFuzzyMatchAnyIndex(FunctionFactory &);
|
||||
void registerFunctionMultiFuzzyMatchAllIndices(FunctionFactory &);
|
||||
void registerFunctionExtractGroups(FunctionFactory &);
|
||||
void registerFunctionExtractAllGroups(FunctionFactory &);
|
||||
void registerFunctionExtractAllGroupsVertical(FunctionFactory &);
|
||||
void registerFunctionExtractAllGroupsHorizontal(FunctionFactory &);
|
||||
|
||||
void registerFunctionsStringRegexp(FunctionFactory & factory)
|
||||
{
|
||||
@ -37,7 +38,8 @@ void registerFunctionsStringRegexp(FunctionFactory & factory)
|
||||
registerFunctionMultiFuzzyMatchAnyIndex(factory);
|
||||
registerFunctionMultiFuzzyMatchAllIndices(factory);
|
||||
registerFunctionExtractGroups(factory);
|
||||
registerFunctionExtractAllGroups(factory);
|
||||
registerFunctionExtractAllGroupsVertical(factory);
|
||||
registerFunctionExtractAllGroupsHorizontal(factory);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -144,7 +144,8 @@ SRCS(
|
||||
exp10.cpp
|
||||
exp2.cpp
|
||||
exp.cpp
|
||||
extractAllGroups.cpp
|
||||
extractAllGroupsHorizontal.cpp
|
||||
extractAllGroupsVertical.cpp
|
||||
extract.cpp
|
||||
extractGroups.cpp
|
||||
extractTimeZoneFromFunctionArguments.cpp
|
||||
|
||||
@ -982,7 +982,16 @@ void Context::setSetting(const StringRef & name, const Field & value)
|
||||
|
||||
void Context::applySettingChange(const SettingChange & change)
|
||||
{
|
||||
try
|
||||
{
|
||||
setSetting(change.name, change.value);
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
e.addMessage(fmt::format("in attempt to set the value of setting '{}' to {}",
|
||||
change.name, applyVisitor(FieldVisitorToString(), change.value)));
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -202,11 +202,11 @@ private:
|
||||
{
|
||||
std::optional<size_t> left_table_pos = IdentifierSemantic::getMembership(left);
|
||||
if (!left_table_pos)
|
||||
left_table_pos = IdentifierSemantic::chooseTable(left, tables);
|
||||
left_table_pos = IdentifierSemantic::chooseTableColumnMatch(left, tables);
|
||||
|
||||
std::optional<size_t> right_table_pos = IdentifierSemantic::getMembership(right);
|
||||
if (!right_table_pos)
|
||||
right_table_pos = IdentifierSemantic::chooseTable(right, tables);
|
||||
right_table_pos = IdentifierSemantic::chooseTableColumnMatch(right, tables);
|
||||
|
||||
if (left_table_pos && right_table_pos && (*left_table_pos != *right_table_pos))
|
||||
{
|
||||
|
||||
@ -422,7 +422,7 @@ void DDLWorker::processTasks()
|
||||
}
|
||||
catch (const Coordination::Exception & e)
|
||||
{
|
||||
if (server_startup && e.code == Coordination::ZNONODE)
|
||||
if (server_startup && e.code == Coordination::Error::ZNONODE)
|
||||
{
|
||||
LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true));
|
||||
}
|
||||
@ -603,15 +603,15 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper)
|
||||
|
||||
auto code = zookeeper->tryCreate(active_node_path, "", zkutil::CreateMode::Ephemeral, dummy);
|
||||
|
||||
if (code == Coordination::ZOK || code == Coordination::ZNODEEXISTS)
|
||||
if (code == Coordination::Error::ZOK || code == Coordination::Error::ZNODEEXISTS)
|
||||
{
|
||||
// Ok
|
||||
}
|
||||
else if (code == Coordination::ZNONODE)
|
||||
else if (code == Coordination::Error::ZNONODE)
|
||||
{
|
||||
/// There is no parent
|
||||
createStatusDirs(task.entry_path, zookeeper);
|
||||
if (Coordination::ZOK != zookeeper->tryCreate(active_node_path, "", zkutil::CreateMode::Ephemeral, dummy))
|
||||
if (Coordination::Error::ZOK != zookeeper->tryCreate(active_node_path, "", zkutil::CreateMode::Ephemeral, dummy))
|
||||
throw Coordination::Exception(code, active_node_path);
|
||||
}
|
||||
else
|
||||
@ -915,8 +915,9 @@ void DDLWorker::createStatusDirs(const std::string & node_path, const ZooKeeperP
|
||||
ops.emplace_back(std::make_shared<Coordination::CreateRequest>(std::move(request)));
|
||||
}
|
||||
Coordination::Responses responses;
|
||||
int code = zookeeper->tryMulti(ops, responses);
|
||||
if (code && code != Coordination::ZNODEEXISTS)
|
||||
Coordination::Error code = zookeeper->tryMulti(ops, responses);
|
||||
if (code != Coordination::Error::ZOK
|
||||
&& code != Coordination::Error::ZNODEEXISTS)
|
||||
throw Coordination::Exception(code);
|
||||
}
|
||||
|
||||
@ -1013,7 +1014,7 @@ void DDLWorker::runMainThread()
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (e.code == Coordination::ZNONODE)
|
||||
else if (e.code == Coordination::Error::ZNONODE)
|
||||
{
|
||||
LOG_ERROR(log, "ZooKeeper error: {}", getCurrentExceptionMessage(true));
|
||||
}
|
||||
@ -1201,8 +1202,8 @@ private:
|
||||
static Strings getChildrenAllowNoNode(const std::shared_ptr<zkutil::ZooKeeper> & zookeeper, const String & node_path)
|
||||
{
|
||||
Strings res;
|
||||
int code = zookeeper->tryGetChildren(node_path, res);
|
||||
if (code && code != Coordination::ZNONODE)
|
||||
Coordination::Error code = zookeeper->tryGetChildren(node_path, res);
|
||||
if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE)
|
||||
throw Coordination::Exception(code, node_path);
|
||||
return res;
|
||||
}
|
||||
|
||||
72
src/Interpreters/DuplicateDistinctVisitor.h
Normal file
72
src/Interpreters/DuplicateDistinctVisitor.h
Normal file
@ -0,0 +1,72 @@
|
||||
#pragma once
|
||||
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Interpreters/InDepthNodeVisitor.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ASTSelectQuery.h>
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
#include <Parsers/ASTTablesInSelectQuery.h>
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Removes duplicate DISTINCT from queries.
|
||||
class DuplicateDistinctMatcher
|
||||
{
|
||||
public:
|
||||
struct Data
|
||||
{
|
||||
bool is_distinct;
|
||||
std::vector<String> last_ids;
|
||||
};
|
||||
|
||||
static void visit(const ASTPtr & ast, Data & data)
|
||||
{
|
||||
auto * select_query = ast->as<ASTSelectQuery>();
|
||||
if (select_query)
|
||||
visit(*select_query, data);
|
||||
}
|
||||
|
||||
static void visit(ASTSelectQuery & select_query, Data & data)
|
||||
{
|
||||
if (!select_query.distinct || !select_query.select())
|
||||
return;
|
||||
|
||||
/// Optimize shouldn't work for distributed tables
|
||||
for (const auto & elem : select_query.children)
|
||||
{
|
||||
if (elem->as<ASTSetQuery>() && !elem->as<ASTSetQuery>()->is_standalone)
|
||||
return;
|
||||
}
|
||||
|
||||
auto expression_list = select_query.select();
|
||||
std::vector<String> current_ids;
|
||||
|
||||
if (expression_list->children.empty())
|
||||
return;
|
||||
|
||||
current_ids.reserve(expression_list->children.size());
|
||||
for (const auto & id : expression_list->children)
|
||||
current_ids.push_back(id->getColumnName());
|
||||
|
||||
if (data.is_distinct && current_ids == data.last_ids)
|
||||
select_query.distinct = false;
|
||||
|
||||
data.is_distinct = true;
|
||||
data.last_ids = std::move(current_ids);
|
||||
}
|
||||
|
||||
static bool needChildVisit(const ASTPtr &, const ASTPtr &)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
using DuplicateDistinctVisitor = InDepthNodeVisitor<DuplicateDistinctMatcher, false>;
|
||||
|
||||
}
|
||||
127
src/Interpreters/DuplicateOrderByVisitor.h
Normal file
127
src/Interpreters/DuplicateOrderByVisitor.h
Normal file
@ -0,0 +1,127 @@
|
||||
#pragma once
|
||||
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Interpreters/InDepthNodeVisitor.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ASTSelectQuery.h>
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
#include <Parsers/ASTTablesInSelectQuery.h>
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Checks if SELECT has stateful functions
|
||||
class ASTFunctionStatefulData
|
||||
{
|
||||
public:
|
||||
using TypeToVisit = ASTFunction;
|
||||
|
||||
const Context & context;
|
||||
bool & is_stateful;
|
||||
void visit(ASTFunction & ast_function, ASTPtr &)
|
||||
{
|
||||
if (ast_function.name == "any" || ast_function.name == "groupArray")
|
||||
{
|
||||
is_stateful = true;
|
||||
return;
|
||||
}
|
||||
|
||||
const auto & function = FunctionFactory::instance().tryGet(ast_function.name, context);
|
||||
|
||||
if (function && function->isStateful())
|
||||
{
|
||||
is_stateful = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
using ASTFunctionStatefulMatcher = OneTypeMatcher<ASTFunctionStatefulData>;
|
||||
using ASTFunctionStatefulVisitor = InDepthNodeVisitor<ASTFunctionStatefulMatcher, true>;
|
||||
|
||||
|
||||
/// Erases unnecessary ORDER BY from subquery
|
||||
class DuplicateOrderByFromSubqueriesData
|
||||
{
|
||||
public:
|
||||
using TypeToVisit = ASTSelectQuery;
|
||||
|
||||
bool done = false;
|
||||
|
||||
void visit(ASTSelectQuery & select_query, ASTPtr &)
|
||||
{
|
||||
if (done)
|
||||
return;
|
||||
|
||||
if (select_query.orderBy() && !select_query.limitBy() && !select_query.limitByOffset() &&
|
||||
!select_query.limitByLength() && !select_query.limitLength() && !select_query.limitOffset())
|
||||
{
|
||||
select_query.setExpression(ASTSelectQuery::Expression::ORDER_BY, nullptr);
|
||||
}
|
||||
|
||||
done = true;
|
||||
}
|
||||
};
|
||||
|
||||
using DuplicateOrderByFromSubqueriesMatcher = OneTypeMatcher<DuplicateOrderByFromSubqueriesData>;
|
||||
using DuplicateOrderByFromSubqueriesVisitor = InDepthNodeVisitor<DuplicateOrderByFromSubqueriesMatcher, true>;
|
||||
|
||||
|
||||
/// Finds SELECT that can be optimized
|
||||
class DuplicateOrderByData
|
||||
{
|
||||
public:
|
||||
using TypeToVisit = ASTSelectQuery;
|
||||
|
||||
const Context & context;
|
||||
bool done = false;
|
||||
|
||||
void visit(ASTSelectQuery & select_query, ASTPtr &)
|
||||
{
|
||||
if (done)
|
||||
return;
|
||||
|
||||
/// Disable optimization for distributed tables
|
||||
for (const auto & elem : select_query.children)
|
||||
{
|
||||
if (elem->as<ASTSetQuery>() && !elem->as<ASTSetQuery>()->is_standalone)
|
||||
return;
|
||||
}
|
||||
|
||||
if (select_query.orderBy() || select_query.groupBy())
|
||||
{
|
||||
for (auto & elem : select_query.children)
|
||||
{
|
||||
if (elem->as<ASTExpressionList>())
|
||||
{
|
||||
bool is_stateful = false;
|
||||
ASTFunctionStatefulVisitor::Data data{context, is_stateful};
|
||||
ASTFunctionStatefulVisitor(data).visit(elem);
|
||||
if (is_stateful)
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (auto select_table_ptr = select_query.tables())
|
||||
{
|
||||
if (auto * select_table = select_table_ptr->as<ASTTablesInSelectQuery>())
|
||||
{
|
||||
if (!select_table->children.empty())
|
||||
{
|
||||
DuplicateOrderByFromSubqueriesVisitor::Data data{false};
|
||||
DuplicateOrderByFromSubqueriesVisitor(data).visit(select_table->children[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
using DuplicateOrderByMatcher = OneTypeMatcher<DuplicateOrderByData>;
|
||||
using DuplicateOrderByVisitor = InDepthNodeVisitor<DuplicateOrderByMatcher, true>;
|
||||
|
||||
}
|
||||
@ -208,7 +208,9 @@ struct ExpressionAnalysisResult
|
||||
const FilterInfoPtr & filter_info,
|
||||
const Block & source_header);
|
||||
|
||||
/// Filter for row-level security.
|
||||
bool hasFilter() const { return filter_info.get(); }
|
||||
|
||||
bool hasJoin() const { return before_join.get(); }
|
||||
bool hasPrewhere() const { return prewhere_info.get(); }
|
||||
bool hasWhere() const { return before_where.get(); }
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
#include <Interpreters/FillingRow.h>
|
||||
#include <Common/FieldVisitorsAccurateComparison.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
#include <Core/SortDescription.h>
|
||||
#include <Columns/IColumn.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -16,7 +16,8 @@ namespace
|
||||
{
|
||||
|
||||
template <typename T>
|
||||
std::optional<size_t> tryChooseTable(const ASTIdentifier & identifier, const std::vector<T> & tables, bool allow_ambiguous)
|
||||
std::optional<size_t> tryChooseTable(const ASTIdentifier & identifier, const std::vector<T> & tables,
|
||||
bool allow_ambiguous, bool column_match [[maybe_unused]] = false)
|
||||
{
|
||||
using ColumnMatch = IdentifierSemantic::ColumnMatch;
|
||||
|
||||
@ -27,6 +28,13 @@ std::optional<size_t> tryChooseTable(const ASTIdentifier & identifier, const std
|
||||
for (size_t i = 0; i < tables.size(); ++i)
|
||||
{
|
||||
auto match = IdentifierSemantic::canReferColumnToTable(identifier, tables[i]);
|
||||
|
||||
if constexpr (std::is_same_v<T, TableWithColumnNamesAndTypes>)
|
||||
{
|
||||
if (column_match && match == ColumnMatch::NoMatch && identifier.isShort() && tables[i].hasColumn(identifier.shortName()))
|
||||
match = ColumnMatch::ColumnName;
|
||||
}
|
||||
|
||||
if (match != ColumnMatch::NoMatch)
|
||||
{
|
||||
if (match > best_match)
|
||||
@ -125,12 +133,17 @@ std::optional<size_t> IdentifierSemantic::chooseTable(const ASTIdentifier & iden
|
||||
return tryChooseTable<DatabaseAndTableWithAlias>(identifier, tables, ambiguous);
|
||||
}
|
||||
|
||||
std::optional<size_t> IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const std::vector<TableWithColumnNamesAndTypes> & tables,
|
||||
bool ambiguous)
|
||||
std::optional<size_t> IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const TablesWithColumns & tables, bool ambiguous)
|
||||
{
|
||||
return tryChooseTable<TableWithColumnNamesAndTypes>(identifier, tables, ambiguous);
|
||||
}
|
||||
|
||||
std::optional<size_t> IdentifierSemantic::chooseTableColumnMatch(const ASTIdentifier & identifier, const TablesWithColumns & tables,
|
||||
bool ambiguous)
|
||||
{
|
||||
return tryChooseTable<TableWithColumnNamesAndTypes>(identifier, tables, ambiguous, true);
|
||||
}
|
||||
|
||||
StorageID IdentifierSemantic::extractDatabaseAndTable(const ASTIdentifier & identifier)
|
||||
{
|
||||
if (identifier.name_parts.size() > 2)
|
||||
@ -191,14 +204,9 @@ IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const
|
||||
}
|
||||
|
||||
IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const ASTIdentifier & identifier,
|
||||
const TableWithColumnNamesAndTypes & db_and_table)
|
||||
const TableWithColumnNamesAndTypes & table_with_columns)
|
||||
{
|
||||
ColumnMatch match = canReferColumnToTable(identifier, db_and_table.table);
|
||||
#if 0
|
||||
if (match == ColumnMatch::NoMatch && identifier.isShort() && db_and_table.hasColumn(identifier.shortName()))
|
||||
match = ColumnMatch::ColumnName;
|
||||
#endif
|
||||
return match;
|
||||
return canReferColumnToTable(identifier, table_with_columns.table);
|
||||
}
|
||||
|
||||
/// Strip qualificators from left side of column name.
|
||||
|
||||
@ -41,7 +41,7 @@ struct IdentifierSemantic
|
||||
static std::optional<String> extractNestedName(const ASTIdentifier & identifier, const String & table_name);
|
||||
|
||||
static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table);
|
||||
static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNamesAndTypes & db_and_table);
|
||||
static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNamesAndTypes & table_with_columns);
|
||||
|
||||
static void setColumnShortName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table);
|
||||
static void setColumnLongName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table);
|
||||
@ -52,7 +52,9 @@ struct IdentifierSemantic
|
||||
static std::optional<size_t> getMembership(const ASTIdentifier & identifier);
|
||||
static std::optional<size_t> chooseTable(const ASTIdentifier &, const std::vector<DatabaseAndTableWithAlias> & tables,
|
||||
bool allow_ambiguous = false);
|
||||
static std::optional<size_t> chooseTable(const ASTIdentifier &, const std::vector<TableWithColumnNamesAndTypes> & tables,
|
||||
static std::optional<size_t> chooseTable(const ASTIdentifier &, const TablesWithColumns & tables,
|
||||
bool allow_ambiguous = false);
|
||||
static std::optional<size_t> chooseTableColumnMatch(const ASTIdentifier &, const TablesWithColumns & tables,
|
||||
bool allow_ambiguous = false);
|
||||
|
||||
private:
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
#include <Common/typeid_cast.h>
|
||||
|
||||
#include <Core/Defines.h>
|
||||
#include <Core/Settings.h>
|
||||
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
@ -71,6 +72,7 @@ namespace ErrorCodes
|
||||
extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE;
|
||||
extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY;
|
||||
extern const int DICTIONARY_ALREADY_EXISTS;
|
||||
extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE;
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
}
|
||||
|
||||
@ -276,6 +278,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
|
||||
|
||||
/** all default_expressions as a single expression list,
|
||||
* mixed with conversion-columns for each explicitly specified type */
|
||||
|
||||
ASTPtr default_expr_list = std::make_shared<ASTExpressionList>();
|
||||
NamesAndTypesList column_names_and_types;
|
||||
|
||||
@ -284,9 +287,23 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
|
||||
const auto & col_decl = ast->as<ASTColumnDeclaration &>();
|
||||
|
||||
DataTypePtr column_type = nullptr;
|
||||
|
||||
if (col_decl.type)
|
||||
{
|
||||
column_type = DataTypeFactory::instance().get(col_decl.type);
|
||||
|
||||
if (col_decl.null_modifier)
|
||||
{
|
||||
if (column_type->isNullable())
|
||||
throw Exception("Cant use [NOT] NULL modifier with Nullable type", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE);
|
||||
if (*col_decl.null_modifier)
|
||||
column_type = makeNullable(column_type);
|
||||
}
|
||||
else if (context.getSettingsRef().data_type_default_nullable)
|
||||
{
|
||||
column_type = makeNullable(column_type);
|
||||
}
|
||||
|
||||
column_names_and_types.emplace_back(col_decl.name, column_type);
|
||||
}
|
||||
else
|
||||
|
||||
@ -4,20 +4,15 @@
|
||||
#include <DataStreams/OneBlockInputStream.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Interpreters/InDepthNodeVisitor.h>
|
||||
#include <Interpreters/IdentifierSemantic.h>
|
||||
#include <Interpreters/getTableExpressions.h>
|
||||
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
|
||||
#include <Interpreters/InterpreterSelectQuery.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Parsers/DumpASTNode.h>
|
||||
#include <Parsers/ASTSubquery.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTExplainQuery.h>
|
||||
#include <Parsers/ASTTablesInSelectQuery.h>
|
||||
#include <Parsers/ASTSelectWithUnionQuery.h>
|
||||
#include <Parsers/ASTSelectQuery.h>
|
||||
|
||||
#include <Core/Field.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Storages/StorageView.h>
|
||||
#include <sstream>
|
||||
|
||||
@ -31,56 +26,30 @@ namespace
|
||||
{
|
||||
struct Data
|
||||
{
|
||||
bool analyzed = false;
|
||||
const Context & context;
|
||||
};
|
||||
|
||||
static bool needChildVisit(ASTPtr &, ASTPtr &) { return true; }
|
||||
static bool needChildVisit(ASTPtr & node, ASTPtr &)
|
||||
{
|
||||
return !node->as<ASTSelectQuery>();
|
||||
}
|
||||
|
||||
static void visit(ASTPtr & ast, Data & data)
|
||||
{
|
||||
if (auto * select_query = ast->as<ASTSelectQuery>())
|
||||
visit(*select_query, ast, data);
|
||||
if (auto * union_select_query = ast->as<ASTSelectWithUnionQuery>())
|
||||
visit(*union_select_query, ast, data);
|
||||
if (auto * select = ast->as<ASTSelectQuery>())
|
||||
visit(*select, ast, data);
|
||||
}
|
||||
|
||||
static void visit(ASTSelectQuery & select_query, ASTPtr &, Data & data)
|
||||
static void visit(ASTSelectQuery & select, ASTPtr & node, Data & data)
|
||||
{
|
||||
if (!select_query.tables())
|
||||
return;
|
||||
|
||||
for (const auto & child : select_query.tables()->children)
|
||||
{
|
||||
auto * tables_element = child->as<ASTTablesInSelectQueryElement>();
|
||||
|
||||
if (tables_element && tables_element->table_expression)
|
||||
visit(*tables_element->table_expression->as<ASTTableExpression>(), select_query, data);
|
||||
}
|
||||
}
|
||||
|
||||
static void visit(ASTSelectWithUnionQuery &, ASTPtr & node, Data & data)
|
||||
{
|
||||
if (!data.analyzed)
|
||||
{
|
||||
data.analyzed = true;
|
||||
InterpreterSelectWithUnionQuery interpreter(
|
||||
InterpreterSelectQuery interpreter(
|
||||
node, data.context, SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze().modify());
|
||||
}
|
||||
}
|
||||
|
||||
static void visit(ASTTableExpression & expression, ASTSelectQuery & select_query, Data & data)
|
||||
const SelectQueryInfo & query_info = interpreter.getQueryInfo();
|
||||
if (query_info.view_query)
|
||||
{
|
||||
if (data.context.getSettingsRef().enable_optimize_predicate_expression && expression.database_and_table_name)
|
||||
{
|
||||
if (const auto * identifier = expression.database_and_table_name->as<ASTIdentifier>())
|
||||
{
|
||||
auto table_id = data.context.resolveStorageID(*identifier);
|
||||
const auto & storage = DatabaseCatalog::instance().getTable(table_id, data.context);
|
||||
|
||||
if (auto * storage_view = dynamic_cast<StorageView *>(storage.get()))
|
||||
storage_view->getRuntimeViewQuery(&select_query, data.context, true);
|
||||
}
|
||||
ASTPtr tmp;
|
||||
StorageView::replaceWithSubquery(select, query_info.view_query->clone(), tmp);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@ -30,6 +30,7 @@
|
||||
#include <Storages/MergeTree/MergeTreeData.h>
|
||||
#include <Storages/MergeTree/MergeTreeWhereOptimizer.h>
|
||||
#include <Storages/IStorage.h>
|
||||
#include <Storages/StorageView.h>
|
||||
|
||||
#include <TableFunctions/ITableFunction.h>
|
||||
|
||||
@ -37,7 +38,7 @@
|
||||
#include <Core/Field.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Columns/Collator.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/FieldVisitorsAccurateComparison.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/checkStackSize.h>
|
||||
#include <ext/map.h>
|
||||
@ -94,7 +95,8 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
/// Assumes `storage` is set and the table filter (row-level security) is not empty.
|
||||
String InterpreterSelectQuery::generateFilterActions(ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns) const
|
||||
String InterpreterSelectQuery::generateFilterActions(
|
||||
ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns) const
|
||||
{
|
||||
const auto & db_name = table_id.getDatabaseName();
|
||||
const auto & table_name = table_id.getTableName();
|
||||
@ -185,6 +187,26 @@ static Context getSubqueryContext(const Context & context)
|
||||
return subquery_context;
|
||||
}
|
||||
|
||||
static void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & tables, const String & database, const Settings & settings)
|
||||
{
|
||||
ASTSelectQuery & select = query->as<ASTSelectQuery &>();
|
||||
|
||||
Aliases aliases;
|
||||
if (ASTPtr with = select.with())
|
||||
QueryAliasesNoSubqueriesVisitor(aliases).visit(with);
|
||||
QueryAliasesNoSubqueriesVisitor(aliases).visit(select.select());
|
||||
|
||||
CrossToInnerJoinVisitor::Data cross_to_inner{tables, aliases, database};
|
||||
CrossToInnerJoinVisitor(cross_to_inner).visit(query);
|
||||
|
||||
size_t rewriter_version = settings.multiple_joins_rewriter_version;
|
||||
if (!rewriter_version || rewriter_version > 2)
|
||||
throw Exception("Bad multiple_joins_rewriter_version setting value: " + settings.multiple_joins_rewriter_version.toString(),
|
||||
ErrorCodes::INVALID_SETTING_VALUE);
|
||||
JoinToSubqueryTransformVisitor::Data join_to_subs_data{tables, aliases, rewriter_version};
|
||||
JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query);
|
||||
}
|
||||
|
||||
InterpreterSelectQuery::InterpreterSelectQuery(
|
||||
const ASTPtr & query_ptr_,
|
||||
const Context & context_,
|
||||
@ -241,29 +263,14 @@ InterpreterSelectQuery::InterpreterSelectQuery(
|
||||
/// Rewrite JOINs
|
||||
if (!has_input && joined_tables.tablesCount() > 1)
|
||||
{
|
||||
ASTSelectQuery & select = getSelectQuery();
|
||||
rewriteMultipleJoins(query_ptr, joined_tables.tablesWithColumns(), context->getCurrentDatabase(), settings);
|
||||
|
||||
Aliases aliases;
|
||||
if (ASTPtr with = select.with())
|
||||
QueryAliasesNoSubqueriesVisitor(aliases).visit(with);
|
||||
QueryAliasesNoSubqueriesVisitor(aliases).visit(select.select());
|
||||
|
||||
CrossToInnerJoinVisitor::Data cross_to_inner{joined_tables.tablesWithColumns(), aliases, context->getCurrentDatabase()};
|
||||
CrossToInnerJoinVisitor(cross_to_inner).visit(query_ptr);
|
||||
|
||||
size_t rewriter_version = settings.multiple_joins_rewriter_version;
|
||||
if (!rewriter_version || rewriter_version > 2)
|
||||
throw Exception("Bad multiple_joins_rewriter_version setting value: " + settings.multiple_joins_rewriter_version.toString(),
|
||||
ErrorCodes::INVALID_SETTING_VALUE);
|
||||
JoinToSubqueryTransformVisitor::Data join_to_subs_data{joined_tables.tablesWithColumns(), aliases, rewriter_version};
|
||||
JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query_ptr);
|
||||
|
||||
joined_tables.reset(select);
|
||||
joined_tables.reset(getSelectQuery());
|
||||
joined_tables.resolveTables();
|
||||
|
||||
if (storage && joined_tables.isLeftTableSubquery())
|
||||
{
|
||||
/// Rewritten with subquery. Free storage here locks here.
|
||||
/// Rewritten with subquery. Free storage locks here.
|
||||
storage = {};
|
||||
table_lock.release();
|
||||
table_id = StorageID::createEmpty();
|
||||
@ -287,12 +294,28 @@ InterpreterSelectQuery::InterpreterSelectQuery(
|
||||
if (storage)
|
||||
row_policy_filter = context->getRowPolicyCondition(table_id.getDatabaseName(), table_id.getTableName(), RowPolicy::SELECT_FILTER);
|
||||
|
||||
StorageView * view = nullptr;
|
||||
if (storage)
|
||||
view = dynamic_cast<StorageView *>(storage.get());
|
||||
|
||||
auto analyze = [&] (bool try_move_to_prewhere)
|
||||
{
|
||||
/// Allow push down and other optimizations for VIEW: replace with subquery and rewrite it.
|
||||
ASTPtr view_table;
|
||||
if (view)
|
||||
view->replaceWithSubquery(getSelectQuery(), view_table);
|
||||
|
||||
syntax_analyzer_result = SyntaxAnalyzer(*context).analyzeSelect(
|
||||
query_ptr, SyntaxAnalyzerResult(source_header.getNamesAndTypesList(), storage),
|
||||
options, joined_tables.tablesWithColumns(), required_result_column_names, table_join);
|
||||
|
||||
if (view)
|
||||
{
|
||||
/// Restore original view name. Save rewritten subquery for future usage in StorageView.
|
||||
query_info.view_query = view->restoreViewName(getSelectQuery(), view_table);
|
||||
view = nullptr;
|
||||
}
|
||||
|
||||
if (try_move_to_prewhere && storage && !row_policy_filter && query.where() && !query.prewhere() && !query.final())
|
||||
{
|
||||
/// PREWHERE optimization: transfer some condition from WHERE to PREWHERE if enabled and viable
|
||||
@ -474,8 +497,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
|
||||
second_stage,
|
||||
options.only_analyze,
|
||||
filter_info,
|
||||
source_header
|
||||
);
|
||||
source_header);
|
||||
|
||||
if (options.to_stage == QueryProcessingStage::Enum::FetchColumns)
|
||||
{
|
||||
@ -979,10 +1001,13 @@ void InterpreterSelectQuery::executeFetchColumns(
|
||||
|
||||
/// Optimization for trivial query like SELECT count() FROM table.
|
||||
bool optimize_trivial_count =
|
||||
syntax_analyzer_result->optimize_trivial_count && storage &&
|
||||
processing_stage == QueryProcessingStage::FetchColumns &&
|
||||
query_analyzer->hasAggregation() && (query_analyzer->aggregates().size() == 1) &&
|
||||
typeid_cast<AggregateFunctionCount *>(query_analyzer->aggregates()[0].function.get());
|
||||
syntax_analyzer_result->optimize_trivial_count
|
||||
&& storage
|
||||
&& !filter_info
|
||||
&& processing_stage == QueryProcessingStage::FetchColumns
|
||||
&& query_analyzer->hasAggregation()
|
||||
&& (query_analyzer->aggregates().size() == 1)
|
||||
&& typeid_cast<AggregateFunctionCount *>(query_analyzer->aggregates()[0].function.get());
|
||||
|
||||
if (optimize_trivial_count)
|
||||
{
|
||||
|
||||
@ -88,6 +88,8 @@ public:
|
||||
|
||||
size_t getMaxStreams() const { return max_streams; }
|
||||
|
||||
const SelectQueryInfo & getQueryInfo() const { return query_info; }
|
||||
|
||||
private:
|
||||
InterpreterSelectQuery(
|
||||
const ASTPtr & query_ptr_,
|
||||
@ -132,7 +134,8 @@ private:
|
||||
void executeSubqueriesInSetsAndJoins(QueryPipeline & pipeline, const std::unordered_map<String, SubqueryForSet> & subqueries_for_sets);
|
||||
void executeMergeSorted(QueryPipeline & pipeline, const SortDescription & sort_description, UInt64 limit);
|
||||
|
||||
String generateFilterActions(ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns = {}) const;
|
||||
String generateFilterActions(
|
||||
ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns = {}) const;
|
||||
|
||||
enum class Modificator
|
||||
{
|
||||
@ -159,6 +162,7 @@ private:
|
||||
|
||||
/// Is calculated in getSampleBlock. Is used later in readImpl.
|
||||
ExpressionAnalysisResult analysis_result;
|
||||
/// For row-level security.
|
||||
FilterInfoPtr filter_info;
|
||||
|
||||
QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns;
|
||||
|
||||
@ -585,8 +585,9 @@ std::vector<TableNeededColumns> normalizeColumnNamesExtractNeeded(
|
||||
for (ASTIdentifier * ident : identifiers)
|
||||
{
|
||||
bool got_alias = aliases.count(ident->name);
|
||||
bool allow_ambiguous = got_alias; /// allow ambiguous column overridden by an alias
|
||||
|
||||
if (auto table_pos = IdentifierSemantic::chooseTable(*ident, tables))
|
||||
if (auto table_pos = IdentifierSemantic::chooseTableColumnMatch(*ident, tables, allow_ambiguous))
|
||||
{
|
||||
if (!ident->isShort())
|
||||
{
|
||||
|
||||
@ -34,7 +34,8 @@ public:
|
||||
void makeFakeTable(StoragePtr storage, const Block & source_header);
|
||||
std::shared_ptr<TableJoin> makeTableJoin(const ASTSelectQuery & select_query);
|
||||
|
||||
const std::vector<TableWithColumnNamesAndTypes> & tablesWithColumns() const { return tables_with_columns; }
|
||||
const TablesWithColumns & tablesWithColumns() const { return tables_with_columns; }
|
||||
TablesWithColumns moveTablesWithColumns() { return std::move(tables_with_columns); }
|
||||
|
||||
bool isLeftTableSubquery() const;
|
||||
bool isLeftTableFunction() const;
|
||||
@ -49,7 +50,7 @@ public:
|
||||
private:
|
||||
Context context;
|
||||
std::vector<const ASTTableExpression *> table_expressions;
|
||||
std::vector<TableWithColumnNamesAndTypes> tables_with_columns;
|
||||
TablesWithColumns tables_with_columns;
|
||||
|
||||
/// Legacy (duplicated left table values)
|
||||
ASTPtr left_table_expression;
|
||||
|
||||
@ -23,12 +23,15 @@
|
||||
#include <Interpreters/getTableExpressions.h>
|
||||
#include <Interpreters/OptimizeIfChains.h>
|
||||
#include <Interpreters/ArithmeticOperationsInAgrFuncOptimize.h>
|
||||
#include <Interpreters/DuplicateDistinctVisitor.h>
|
||||
#include <Interpreters/DuplicateOrderByVisitor.h>
|
||||
|
||||
#include <Parsers/ASTExpressionList.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ASTOrderByElement.h>
|
||||
#include <Parsers/ASTSelectQuery.h>
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
#include <Parsers/ASTTablesInSelectQuery.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
@ -370,6 +373,18 @@ void optimizeOrderBy(const ASTSelectQuery * select_query)
|
||||
elems = std::move(unique_elems);
|
||||
}
|
||||
|
||||
/// Optimize duplicate ORDER BY and DISTINCT
|
||||
void optimizeDuplicateOrderByAndDistinct(ASTPtr & query, bool optimize_duplicate_order_by_and_distinct, const Context & context)
|
||||
{
|
||||
if (optimize_duplicate_order_by_and_distinct)
|
||||
{
|
||||
DuplicateOrderByVisitor::Data order_by_data{context, false};
|
||||
DuplicateOrderByVisitor(order_by_data).visit(query);
|
||||
DuplicateDistinctVisitor::Data distinct_data{};
|
||||
DuplicateDistinctVisitor(distinct_data).visit(query);
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove duplicate items from LIMIT BY.
|
||||
void optimizeLimitBy(const ASTSelectQuery * select_query)
|
||||
{
|
||||
@ -831,6 +846,9 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect(
|
||||
/// Remove duplicate items from ORDER BY.
|
||||
optimizeOrderBy(select_query);
|
||||
|
||||
/// Remove duplicate ORDER BY and DISTINCT from subqueries.
|
||||
optimizeDuplicateOrderByAndDistinct(query, settings.optimize_duplicate_order_by_and_distinct, context);
|
||||
|
||||
/// Remove duplicated elements from LIMIT BY clause.
|
||||
optimizeLimitBy(select_query);
|
||||
|
||||
|
||||
@ -33,8 +33,6 @@ namespace ErrorCodes
|
||||
{
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int TYPE_MISMATCH;
|
||||
extern const int TOO_LARGE_STRING_SIZE;
|
||||
extern const int CANNOT_CONVERT_TYPE;
|
||||
}
|
||||
|
||||
|
||||
@ -124,42 +122,6 @@ static Field convertDecimalType(const Field & from, const To & type)
|
||||
}
|
||||
|
||||
|
||||
DayNum stringToDate(const String & s)
|
||||
{
|
||||
ReadBufferFromString in(s);
|
||||
DayNum date{};
|
||||
|
||||
readDateText(date, in);
|
||||
if (!in.eof())
|
||||
throw Exception("String is too long for Date: " + s, ErrorCodes::TOO_LARGE_STRING_SIZE);
|
||||
|
||||
return date;
|
||||
}
|
||||
|
||||
UInt64 stringToDateTime(const String & s)
|
||||
{
|
||||
ReadBufferFromString in(s);
|
||||
time_t date_time{};
|
||||
|
||||
readDateTimeText(date_time, in);
|
||||
if (!in.eof())
|
||||
throw Exception("String is too long for DateTime: " + s, ErrorCodes::TOO_LARGE_STRING_SIZE);
|
||||
|
||||
return UInt64(date_time);
|
||||
}
|
||||
|
||||
DateTime64::NativeType stringToDateTime64(const String & s, UInt32 scale)
|
||||
{
|
||||
ReadBufferFromString in(s);
|
||||
DateTime64 datetime64 {0};
|
||||
|
||||
readDateTime64Text(datetime64, scale, in);
|
||||
if (!in.eof())
|
||||
throw Exception("String is too long for DateTime64: " + s, ErrorCodes::TOO_LARGE_STRING_SIZE);
|
||||
|
||||
return datetime64.value;
|
||||
}
|
||||
|
||||
Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const IDataType * from_type_hint)
|
||||
{
|
||||
WhichDataType which_type(type);
|
||||
@ -184,7 +146,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
|
||||
{
|
||||
return static_cast<const DataTypeDateTime &>(type).getTimeZone().fromDayNum(DayNum(src.get<UInt64>()));
|
||||
}
|
||||
else if (type.isValueRepresentedByNumber())
|
||||
else if (type.isValueRepresentedByNumber() && src.getType() != Field::Types::String)
|
||||
{
|
||||
if (which_type.isUInt8()) return convertNumericType<UInt8>(src, type);
|
||||
if (which_type.isUInt16()) return convertNumericType<UInt16>(src, type);
|
||||
@ -200,9 +162,6 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
|
||||
if (const auto * ptype = typeid_cast<const DataTypeDecimal<Decimal64> *>(&type)) return convertDecimalType(src, *ptype);
|
||||
if (const auto * ptype = typeid_cast<const DataTypeDecimal<Decimal128> *>(&type)) return convertDecimalType(src, *ptype);
|
||||
|
||||
if (!which_type.isDateOrDateTime() && !which_type.isUUID() && !which_type.isEnum())
|
||||
throw Exception{"Cannot convert field to type " + type.getName(), ErrorCodes::CANNOT_CONVERT_TYPE};
|
||||
|
||||
if (which_type.isEnum() && (src.getType() == Field::Types::UInt64 || src.getType() == Field::Types::Int64))
|
||||
{
|
||||
/// Convert UInt64 or Int64 to Enum's value
|
||||
@ -214,36 +173,20 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
|
||||
/// We don't need any conversion UInt64 is under type of Date and DateTime
|
||||
return src;
|
||||
}
|
||||
// TODO (vnemkov): extra cases for DateTime64: converting from integer, converting from Decimal
|
||||
|
||||
if (src.getType() == Field::Types::String)
|
||||
if (which_type.isUUID() && src.getType() == Field::Types::UInt128)
|
||||
{
|
||||
if (which_type.isDate())
|
||||
/// Already in needed type.
|
||||
return src;
|
||||
}
|
||||
|
||||
if (which_type.isDateTime64() && src.getType() == Field::Types::Decimal64)
|
||||
{
|
||||
/// Convert 'YYYY-MM-DD' Strings to Date
|
||||
return stringToDate(src.get<const String &>());
|
||||
}
|
||||
else if (which_type.isDateTime())
|
||||
{
|
||||
/// Convert 'YYYY-MM-DD hh:mm:ss' Strings to DateTime
|
||||
return stringToDateTime(src.get<const String &>());
|
||||
}
|
||||
else if (which_type.isDateTime64())
|
||||
{
|
||||
const auto * date_time64 = typeid_cast<const DataTypeDateTime64 *>(&type);
|
||||
/// Convert 'YYYY-MM-DD hh:mm:ss.NNNNNNNNN' Strings to DateTime
|
||||
return stringToDateTime64(src.get<const String &>(), date_time64->getScale());
|
||||
}
|
||||
else if (which_type.isUUID())
|
||||
{
|
||||
return stringToUUID(src.get<const String &>());
|
||||
}
|
||||
else if (which_type.isEnum())
|
||||
{
|
||||
/// Convert String to Enum's value
|
||||
return dynamic_cast<const IDataTypeEnum &>(type).castToValue(src);
|
||||
}
|
||||
/// Already in needed type.
|
||||
return src;
|
||||
}
|
||||
|
||||
/// TODO Conversion from integers to DateTime64
|
||||
}
|
||||
else if (which_type.isStringOrFixedString())
|
||||
{
|
||||
@ -328,17 +271,37 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
|
||||
return src;
|
||||
}
|
||||
|
||||
/// Conversion from string by parsing.
|
||||
if (src.getType() == Field::Types::String)
|
||||
{
|
||||
const auto col = type.createColumn();
|
||||
ReadBufferFromString buffer(src.get<String>());
|
||||
type.deserializeAsTextEscaped(*col, buffer, FormatSettings{});
|
||||
/// Promote data type to avoid overflows. Note that overflows in the largest data type are still possible.
|
||||
const IDataType * type_to_parse = &type;
|
||||
DataTypePtr holder;
|
||||
|
||||
return (*col)[0];
|
||||
if (type.canBePromoted())
|
||||
{
|
||||
holder = type.promoteNumericType();
|
||||
type_to_parse = holder.get();
|
||||
}
|
||||
|
||||
const auto col = type_to_parse->createColumn();
|
||||
ReadBufferFromString in_buffer(src.get<String>());
|
||||
try
|
||||
{
|
||||
type_to_parse->deserializeAsWholeText(*col, in_buffer, FormatSettings{});
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
e.addMessage(fmt::format("while converting '{}' to {}", src.get<String>(), type.getName()));
|
||||
throw;
|
||||
}
|
||||
if (!in_buffer.eof())
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH, "Cannot convert string {} to type {}", src.get<String>(), type.getName());
|
||||
|
||||
Field parsed = (*col)[0];
|
||||
return convertFieldToType(parsed, type, from_type_hint);
|
||||
}
|
||||
|
||||
// TODO (nemkov): should we attempt to parse value using or `type.deserializeAsTextEscaped()` type.deserializeAsTextEscaped() ?
|
||||
throw Exception("Type mismatch in IN or VALUES section. Expected: " + type.getName() + ". Got: "
|
||||
+ Field::Types::toString(src.getType()), ErrorCodes::TYPE_MISMATCH);
|
||||
}
|
||||
|
||||
@ -61,6 +61,12 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta
|
||||
type->formatImpl(settings, state, frame);
|
||||
}
|
||||
|
||||
if (null_modifier)
|
||||
{
|
||||
settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "")
|
||||
<< (*null_modifier ? "" : "NOT ") << "NULL" << (settings.hilite ? hilite_none : "");
|
||||
}
|
||||
|
||||
if (default_expression)
|
||||
{
|
||||
settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << default_specifier << (settings.hilite ? hilite_none : "") << ' ';
|
||||
|
||||
@ -13,6 +13,7 @@ class ASTColumnDeclaration : public IAST
|
||||
public:
|
||||
String name;
|
||||
ASTPtr type;
|
||||
std::optional<bool> null_modifier;
|
||||
String default_specifier;
|
||||
ASTPtr default_expression;
|
||||
ASTPtr comment;
|
||||
|
||||
@ -65,7 +65,8 @@ namespace
|
||||
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << authentication_type_name
|
||||
<< (settings.hilite ? IAST::hilite_none : "");
|
||||
if (password)
|
||||
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " BY " << quoteString(*password);
|
||||
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " BY " << (settings.hilite ? IAST::hilite_none : "")
|
||||
<< quoteString(*password);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -157,7 +157,7 @@ bool ParserTablePropertyDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expecte
|
||||
|
||||
ParserIndexDeclaration index_p;
|
||||
ParserConstraintDeclaration constraint_p;
|
||||
ParserColumnDeclaration column_p;
|
||||
ParserColumnDeclaration column_p{true, true};
|
||||
|
||||
ASTPtr new_node = nullptr;
|
||||
|
||||
|
||||
@ -92,7 +92,8 @@ template <typename NameParser>
|
||||
class IParserColumnDeclaration : public IParserBase
|
||||
{
|
||||
public:
|
||||
explicit IParserColumnDeclaration(bool require_type_ = true) : require_type(require_type_)
|
||||
explicit IParserColumnDeclaration(bool require_type_ = true, bool allow_null_modifiers_ = false)
|
||||
: require_type(require_type_), allow_null_modifiers(allow_null_modifiers_)
|
||||
{
|
||||
}
|
||||
|
||||
@ -104,6 +105,7 @@ protected:
|
||||
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
|
||||
|
||||
bool require_type = true;
|
||||
bool allow_null_modifiers = false;
|
||||
};
|
||||
|
||||
using ParserColumnDeclaration = IParserColumnDeclaration<ParserIdentifier>;
|
||||
@ -115,6 +117,8 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
|
||||
NameParser name_parser;
|
||||
ParserIdentifierWithOptionalParameters type_parser;
|
||||
ParserKeyword s_default{"DEFAULT"};
|
||||
ParserKeyword s_null{"NULL"};
|
||||
ParserKeyword s_not{"NOT"};
|
||||
ParserKeyword s_materialized{"MATERIALIZED"};
|
||||
ParserKeyword s_alias{"ALIAS"};
|
||||
ParserKeyword s_comment{"COMMENT"};
|
||||
@ -135,6 +139,7 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
|
||||
*/
|
||||
ASTPtr type;
|
||||
String default_specifier;
|
||||
std::optional<bool> null_modifier;
|
||||
ASTPtr default_expression;
|
||||
ASTPtr comment_expression;
|
||||
ASTPtr codec_expression;
|
||||
@ -163,6 +168,17 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
|
||||
if (require_type && !type && !default_expression)
|
||||
return false; /// reject column name without type
|
||||
|
||||
if (type && allow_null_modifiers)
|
||||
{
|
||||
if (s_not.ignore(pos, expected))
|
||||
{
|
||||
if (!s_null.ignore(pos, expected))
|
||||
return false;
|
||||
null_modifier.emplace(false);
|
||||
}
|
||||
else if (s_null.ignore(pos, expected))
|
||||
null_modifier.emplace(true);
|
||||
}
|
||||
|
||||
if (s_comment.ignore(pos, expected))
|
||||
{
|
||||
@ -193,6 +209,8 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
|
||||
column_declaration->children.push_back(std::move(type));
|
||||
}
|
||||
|
||||
column_declaration->null_modifier = null_modifier;
|
||||
|
||||
if (default_expression)
|
||||
{
|
||||
column_declaration->default_specifier = default_specifier;
|
||||
|
||||
@ -100,9 +100,9 @@ void AggregatingInOrderTransform::consume(Chunk chunk)
|
||||
params->aggregator.createStatesAndFillKeyColumnsWithSingleKey(variants, key_columns, key_begin, res_key_columns);
|
||||
++cur_block_size;
|
||||
}
|
||||
size_t mid = 0;
|
||||
size_t high = 0;
|
||||
size_t low = -1;
|
||||
ssize_t mid = 0;
|
||||
ssize_t high = 0;
|
||||
ssize_t low = -1;
|
||||
/// Will split block into segments with the same key
|
||||
while (key_end != rows)
|
||||
{
|
||||
|
||||
@ -430,6 +430,10 @@ void HTTPHandler::processQuery(
|
||||
|
||||
auto param_could_be_skipped = [&] (const String & name)
|
||||
{
|
||||
/// Empty parameter appears when URL like ?&a=b or a=b&&c=d. Just skip them for user's convenience.
|
||||
if (name.empty())
|
||||
return true;
|
||||
|
||||
if (reserved_param_names.count(name))
|
||||
return true;
|
||||
|
||||
|
||||
@ -1,9 +1,6 @@
|
||||
#include "HTTPHandlerFactory.h"
|
||||
|
||||
#include <re2/re2.h>
|
||||
#include <re2/stringpiece.h>
|
||||
#include <common/find_symbols.h>
|
||||
#include <Poco/StringTokenizer.h>
|
||||
#include <Poco/Util/LayeredConfiguration.h>
|
||||
|
||||
#include "HTTPHandler.h"
|
||||
|
||||
@ -104,13 +104,13 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions(
|
||||
lock_ops.push_back(zkutil::makeCheckRequest(block_numbers_path, partitions_stat.version));
|
||||
|
||||
Coordination::Responses lock_responses;
|
||||
int rc = zookeeper.tryMulti(lock_ops, lock_responses);
|
||||
if (rc == Coordination::ZBADVERSION)
|
||||
Coordination::Error rc = zookeeper.tryMulti(lock_ops, lock_responses);
|
||||
if (rc == Coordination::Error::ZBADVERSION)
|
||||
{
|
||||
LOG_TRACE(&Poco::Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry.");
|
||||
continue;
|
||||
}
|
||||
else if (rc != Coordination::ZOK)
|
||||
else if (rc != Coordination::Error::ZOK)
|
||||
throw Coordination::Exception(rc);
|
||||
|
||||
for (size_t i = 0; i < partitions.size(); ++i)
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
#include <Interpreters/misc.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/FieldVisitorsAccurateComparison.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Interpreters/convertFieldToType.h>
|
||||
#include <Interpreters/Set.h>
|
||||
|
||||
@ -82,7 +82,6 @@ public:
|
||||
const AllowedMergingPredicate & can_merge,
|
||||
String * out_disable_reason = nullptr);
|
||||
|
||||
|
||||
/** Select all the parts in the specified partition for merge, if possible.
|
||||
* final - choose to merge even a single part - that is, allow to merge one part "with itself".
|
||||
*/
|
||||
|
||||
@ -905,7 +905,9 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r
|
||||
if (prewhere->remove_prewhere_column)
|
||||
result.columns.erase(result.columns.begin() + prewhere_column_pos);
|
||||
else
|
||||
result.columns[prewhere_column_pos] = DataTypeUInt8().createColumnConst(result.num_rows, 1u)->convertToFullColumnIfConst();
|
||||
result.columns[prewhere_column_pos] =
|
||||
getSampleBlock().getByName(prewhere->prewhere_column_name).type->
|
||||
createColumnConst(result.num_rows, 1u)->convertToFullColumnIfConst();
|
||||
}
|
||||
}
|
||||
/// Filter in WHERE instead
|
||||
|
||||
@ -85,7 +85,7 @@ void ReplicatedMergeTreeBlockOutputStream::checkQuorumPrecondition(zkutil::ZooKe
|
||||
*/
|
||||
|
||||
auto quorum_status = quorum_status_future.get();
|
||||
if (quorum_status.error != Coordination::ZNONODE)
|
||||
if (quorum_status.error != Coordination::Error::ZNONODE)
|
||||
throw Exception("Quorum for previous write has not been satisfied yet. Status: " + quorum_status.data, ErrorCodes::UNSATISFIED_QUORUM_FOR_PREVIOUS_WRITE);
|
||||
|
||||
/// Both checks are implicitly made also later (otherwise there would be a race condition).
|
||||
@ -93,7 +93,7 @@ void ReplicatedMergeTreeBlockOutputStream::checkQuorumPrecondition(zkutil::ZooKe
|
||||
auto is_active = is_active_future.get();
|
||||
auto host = host_future.get();
|
||||
|
||||
if (is_active.error == Coordination::ZNONODE || host.error == Coordination::ZNONODE)
|
||||
if (is_active.error == Coordination::Error::ZNONODE || host.error == Coordination::Error::ZNONODE)
|
||||
throw Exception("Replica is not active right now", ErrorCodes::READONLY);
|
||||
|
||||
quorum_info.is_active_node_value = is_active.data;
|
||||
@ -198,7 +198,8 @@ void ReplicatedMergeTreeBlockOutputStream::writeExistingPart(MergeTreeData::Muta
|
||||
}
|
||||
|
||||
|
||||
void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const String & block_id)
|
||||
void ReplicatedMergeTreeBlockOutputStream::commitPart(
|
||||
zkutil::ZooKeeperPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const String & block_id)
|
||||
{
|
||||
storage.check(part->getColumns());
|
||||
assertSessionIsNotExpired(zookeeper);
|
||||
@ -299,9 +300,9 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
|
||||
storage.renameTempPartAndAdd(part, nullptr, &transaction);
|
||||
|
||||
Coordination::Responses responses;
|
||||
int32_t multi_code = zookeeper->tryMultiNoThrow(ops, responses); /// 1 RTT
|
||||
Coordination::Error multi_code = zookeeper->tryMultiNoThrow(ops, responses); /// 1 RTT
|
||||
|
||||
if (multi_code == Coordination::ZOK)
|
||||
if (multi_code == Coordination::Error::ZOK)
|
||||
{
|
||||
transaction.commit();
|
||||
storage.merge_selecting_task->schedule();
|
||||
@ -309,8 +310,8 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
|
||||
/// Lock nodes have been already deleted, do not delete them in destructor
|
||||
block_number_lock->assumeUnlocked();
|
||||
}
|
||||
else if (multi_code == Coordination::ZCONNECTIONLOSS
|
||||
|| multi_code == Coordination::ZOPERATIONTIMEOUT)
|
||||
else if (multi_code == Coordination::Error::ZCONNECTIONLOSS
|
||||
|| multi_code == Coordination::Error::ZOPERATIONTIMEOUT)
|
||||
{
|
||||
/** If the connection is lost, and we do not know if the changes were applied, we can not delete the local part
|
||||
* if the changes were applied, the inserted block appeared in `/blocks/`, and it can not be inserted again.
|
||||
@ -326,7 +327,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
|
||||
{
|
||||
String failed_op_path = zkutil::KeeperMultiException(multi_code, ops, responses).getPathForFirstFailedOp();
|
||||
|
||||
if (multi_code == Coordination::ZNODEEXISTS && deduplicate_block && failed_op_path == block_id_path)
|
||||
if (multi_code == Coordination::Error::ZNODEEXISTS && deduplicate_block && failed_op_path == block_id_path)
|
||||
{
|
||||
/// Block with the same id have just appeared in table (or other replica), rollback thee insertion.
|
||||
LOG_INFO(log, "Block with ID {} already exists; ignoring it (removing part {})", block_id, part->name);
|
||||
@ -336,7 +337,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
|
||||
last_block_is_duplicate = true;
|
||||
ProfileEvents::increment(ProfileEvents::DuplicatedInsertedBlocks);
|
||||
}
|
||||
else if (multi_code == Coordination::ZNODEEXISTS && failed_op_path == quorum_info.status_path)
|
||||
else if (multi_code == Coordination::Error::ZNODEEXISTS && failed_op_path == quorum_info.status_path)
|
||||
{
|
||||
transaction.rollback();
|
||||
|
||||
@ -347,7 +348,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
|
||||
/// NOTE: We could be here if the node with the quorum existed, but was quickly removed.
|
||||
transaction.rollback();
|
||||
throw Exception("Unexpected logical error while adding block " + toString(block_number) + " with ID '" + block_id + "': "
|
||||
+ zkutil::ZooKeeper::error2string(multi_code) + ", path " + failed_op_path,
|
||||
+ Coordination::errorMessage(multi_code) + ", path " + failed_op_path,
|
||||
ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR);
|
||||
}
|
||||
}
|
||||
@ -355,13 +356,13 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
|
||||
{
|
||||
transaction.rollback();
|
||||
throw Exception("Unrecoverable network error while adding block " + toString(block_number) + " with ID '" + block_id + "': "
|
||||
+ zkutil::ZooKeeper::error2string(multi_code), ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR);
|
||||
+ Coordination::errorMessage(multi_code), ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR);
|
||||
}
|
||||
else
|
||||
{
|
||||
transaction.rollback();
|
||||
throw Exception("Unexpected ZooKeeper error while adding block " + toString(block_number) + " with ID '" + block_id + "': "
|
||||
+ zkutil::ZooKeeper::error2string(multi_code), ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR);
|
||||
+ Coordination::errorMessage(multi_code), ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR);
|
||||
}
|
||||
|
||||
if (quorum)
|
||||
|
||||
@ -40,7 +40,7 @@ void ReplicatedMergeTreeCleanupThread::run()
|
||||
{
|
||||
tryLogCurrentException(log, __PRETTY_FUNCTION__);
|
||||
|
||||
if (e.code == Coordination::ZSESSIONEXPIRED)
|
||||
if (e.code == Coordination::Error::ZSESSIONEXPIRED)
|
||||
return;
|
||||
}
|
||||
catch (...)
|
||||
@ -319,15 +319,15 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks()
|
||||
for (auto & pair : try_remove_futures)
|
||||
{
|
||||
const String & path = pair.first;
|
||||
int32_t rc = pair.second.get().error;
|
||||
if (rc == Coordination::ZNOTEMPTY)
|
||||
Coordination::Error rc = pair.second.get().error;
|
||||
if (rc == Coordination::Error::ZNOTEMPTY)
|
||||
{
|
||||
/// Can happen if there are leftover block nodes with children created by previous server versions.
|
||||
zookeeper->removeRecursive(path);
|
||||
cached_block_stats.erase(first_outdated_block->node);
|
||||
}
|
||||
else if (rc)
|
||||
LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, zkutil::ZooKeeper::error2string(rc));
|
||||
else if (rc != Coordination::Error::ZOK)
|
||||
LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, Coordination::errorMessage(rc));
|
||||
else
|
||||
{
|
||||
/// Successfully removed blocks have to be removed from cache
|
||||
@ -348,7 +348,7 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper &
|
||||
|
||||
Strings blocks;
|
||||
Coordination::Stat stat;
|
||||
if (zookeeper.tryGetChildren(storage.zookeeper_path + "/blocks", blocks, &stat))
|
||||
if (Coordination::Error::ZOK != zookeeper.tryGetChildren(storage.zookeeper_path + "/blocks", blocks, &stat))
|
||||
throw Exception(storage.zookeeper_path + "/blocks doesn't exist", ErrorCodes::NOT_FOUND_NODE);
|
||||
|
||||
/// Seems like this code is obsolete, because we delete blocks from cache
|
||||
@ -391,7 +391,7 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper &
|
||||
for (auto & elem : exists_futures)
|
||||
{
|
||||
auto status = elem.second.get();
|
||||
if (status.error != Coordination::ZNONODE)
|
||||
if (status.error != Coordination::Error::ZNONODE)
|
||||
{
|
||||
cached_block_stats.emplace(elem.first, status.stat.ctime);
|
||||
timed_blocks.emplace_back(elem.first, status.stat.ctime);
|
||||
|
||||
@ -368,7 +368,7 @@ void ReplicatedMergeTreePartCheckThread::run()
|
||||
{
|
||||
tryLogCurrentException(log, __PRETTY_FUNCTION__);
|
||||
|
||||
if (e.code == Coordination::ZSESSIONEXPIRED)
|
||||
if (e.code == Coordination::Error::ZSESSIONEXPIRED)
|
||||
return;
|
||||
|
||||
task->scheduleAfter(PART_CHECK_ERROR_SLEEP_MS);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user