mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-17 13:13:36 +00:00
Merge branch 'master' of https://github.com/ClickHouse/ClickHouse into master
This commit is contained in:
George
commit
5111aa3e93
@ -102,11 +102,11 @@ else
|
||||
echo "No failed tests"
|
||||
fi
|
||||
|
||||
mkdir -p $COVERAGE_DIR
|
||||
mv /*.profraw $COVERAGE_DIR
|
||||
mkdir -p "$COVERAGE_DIR"
|
||||
mv /*.profraw "$COVERAGE_DIR"
|
||||
|
||||
mkdir -p $SOURCE_DIR/obj-x86_64-linux-gnu
|
||||
cd $SOURCE_DIR/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd /
|
||||
llvm-profdata-11 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata
|
||||
llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov
|
||||
genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR}
|
||||
mkdir -p "$SOURCE_DIR"/obj-x86_64-linux-gnu
|
||||
cd "$SOURCE_DIR"/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd /
|
||||
llvm-profdata-11 merge -sparse "${COVERAGE_DIR}"/* -o clickhouse.profdata
|
||||
llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex "$IGNORE" > output.lcov
|
||||
genhtml output.lcov --ignore-errors source --output-directory "${OUTPUT_DIR}"
|
||||
|
||||
@ -65,7 +65,7 @@ function start_server
|
||||
{
|
||||
set -m # Spawn server in its own process groups
|
||||
local opts=(
|
||||
--config-file="$FASTTEST_DATA/config.xml"
|
||||
--config-file "$FASTTEST_DATA/config.xml"
|
||||
--
|
||||
--path "$FASTTEST_DATA"
|
||||
--user_files_path "$FASTTEST_DATA/user_files"
|
||||
|
||||
@ -55,9 +55,9 @@ function run_tests()
|
||||
ADDITIONAL_OPTIONS+=('00000_no_tests_to_skip')
|
||||
fi
|
||||
|
||||
for i in $(seq 1 $NUM_TRIES); do
|
||||
for _ in $(seq 1 "$NUM_TRIES"); do
|
||||
clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt
|
||||
if [ ${PIPESTATUS[0]} -ne "0" ]; then
|
||||
if [ "${PIPESTATUS[0]}" -ne "0" ]; then
|
||||
break;
|
||||
fi
|
||||
done
|
||||
@ -65,4 +65,4 @@ function run_tests()
|
||||
|
||||
export -f run_tests
|
||||
|
||||
timeout $MAX_RUN_TIME bash -c run_tests ||:
|
||||
timeout "$MAX_RUN_TIME" bash -c run_tests ||:
|
||||
|
||||
@ -8,4 +8,5 @@ CMD cd /ClickHouse/utils/check-style && \
|
||||
./check-style -n | tee /test_output/style_output.txt && \
|
||||
./check-typos | tee /test_output/typos_output.txt && \
|
||||
./check-whitespaces -n | tee /test_output/whitespaces_output.txt && \
|
||||
./check-duplicate-includes.sh | tee /test_output/duplicate_output.txt
|
||||
./check-duplicate-includes.sh | tee /test_output/duplicate_output.txt && \
|
||||
./shellcheck-run.sh | tee /test_output/shellcheck_output.txt
|
||||
|
||||
@ -2470,6 +2470,17 @@ Possible values:
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
## data_type_default_nullable {#data_type_default_nullable}
|
||||
|
||||
Allows data types without explicit modifiers [NULL or NOT NULL](../../sql-reference/statements/create/table.md#null-modifiers) in column definition will be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable).
|
||||
|
||||
Possible values:
|
||||
|
||||
- 1 — The data types in column definitions are set to `Nullable` by default.
|
||||
- 0 — The data types in column definitions are set to not `Nullable` by default.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold}
|
||||
|
||||
Enables special logic to perform merges on replicas.
|
||||
|
||||
@ -558,4 +558,46 @@ Result:
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## encodeXMLComponent {#encode-xml-component}
|
||||
|
||||
Escapes characters to place string into XML text node or attribute.
|
||||
|
||||
The following five XML predefined entities will be replaced: `<`, `&`, `>`, `"`, `'`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
encodeXMLComponent(x)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x` — The sequence of characters. [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Returned value(s)**
|
||||
|
||||
- The sequence of characters with escape characters.
|
||||
|
||||
Type: [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT encodeXMLComponent('Hello, "world"!');
|
||||
SELECT encodeXMLComponent('<123>');
|
||||
SELECT encodeXMLComponent('&clickhouse');
|
||||
SELECT encodeXMLComponent('\'foo\'');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
Hello, "world"!
|
||||
<123>
|
||||
&clickhouse
|
||||
'foo'
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_functions/) <!--hide-->
|
||||
|
||||
@ -400,7 +400,8 @@ Result:
|
||||
└──────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**See also**
|
||||
**See Also**
|
||||
|
||||
- [extractAllGroupsVertical](#extractallgroups-vertical)
|
||||
|
||||
## extractAllGroupsVertical {#extractallgroups-vertical}
|
||||
@ -440,7 +441,8 @@ Result:
|
||||
└────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**See also**
|
||||
**See Also**
|
||||
|
||||
- [extractAllGroupsHorizontal](#extractallgroups-horizontal)
|
||||
|
||||
## like(haystack, pattern), haystack LIKE pattern operator {#function-like}
|
||||
@ -590,8 +592,55 @@ Result:
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_search_functions/) <!--hide-->
|
||||
|
||||
## countMatches(haystack, pattern) {#countmatcheshaystack-pattern}
|
||||
|
||||
Returns the number of regular expression matches for a `pattern` in a `haystack`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
countMatches(haystack, pattern)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `pattern` — The regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The number of matches.
|
||||
|
||||
Type: [UInt64](../../sql-reference/data-types/int-uint.md).
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT countMatches('foobar.com', 'o+');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─countMatches('foobar.com', 'o+')─┐
|
||||
│ 2 │
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT countMatches('aaaa', 'aa');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─countMatches('aaaa', 'aa')────┐
|
||||
│ 2 │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_search_functions/) <!--hide-->
|
||||
|
||||
@ -16,8 +16,8 @@ By default, tables are created only on the current server. Distributed DDL queri
|
||||
``` sql
|
||||
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
(
|
||||
name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1],
|
||||
name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [compression_codec] [TTL expr2],
|
||||
name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1],
|
||||
name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr2] [compression_codec] [TTL expr2],
|
||||
...
|
||||
) ENGINE = engine
|
||||
```
|
||||
@ -57,6 +57,14 @@ In all cases, if `IF NOT EXISTS` is specified, the query won’t return an error
|
||||
|
||||
There can be other clauses after the `ENGINE` clause in the query. See detailed documentation on how to create tables in the descriptions of [table engines](../../../engines/table-engines/index.md#table_engines).
|
||||
|
||||
## NULL Or NOT NULL Modifiers {#null-modifiers}
|
||||
|
||||
`NULL` and `NOT NULL` modifiers after data type in column definition allow or do not allow it to be [Nullable](../../../sql-reference/data-types/nullable.md#data_type-nullable).
|
||||
|
||||
If the type is not `Nullable` and if `NULL` is specified, it will be treated as `Nullable`; if `NOT NULL` is specified, then no. For example, `INT NULL` is the same as `Nullable(INT)`. If the type is `Nullable` and `NULL` or `NOT NULL` modifiers are specified, the exception will be thrown.
|
||||
|
||||
See also [data_type_default_nullable](../../../operations/settings/settings.md#data_type_default_nullable) setting.
|
||||
|
||||
## Default Values {#create-default-values}
|
||||
|
||||
The column description can specify an expression for a default value, in one of the following ways: `DEFAULT expr`, `MATERIALIZED expr`, `ALIAS expr`.
|
||||
|
||||
@ -116,12 +116,14 @@ FROM dt
|
||||
|
||||
## See Also {#see-also}
|
||||
|
||||
- [Функции преобразования типов](../../sql-reference/data-types/datetime.md)
|
||||
- [Функции для работы с датой и временем](../../sql-reference/data-types/datetime.md)
|
||||
- [Функции для работы с массивами](../../sql-reference/data-types/datetime.md)
|
||||
- [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format)
|
||||
- [Конфигурационный параметр сервера `timezone`](../../sql-reference/data-types/datetime.md#server_configuration_parameters-timezone)
|
||||
- [Операторы для работы с датой и временем](../../sql-reference/data-types/datetime.md#operators-datetime)
|
||||
- [Функции преобразования типов](../../sql-reference/functions/type-conversion-functions.md)
|
||||
- [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md)
|
||||
- [Функции для работы с массивами](../../sql-reference/functions/array-functions.md)
|
||||
- [Настройка `date_time_input_format`](../../operations/settings/settings/#settings-date_time_input_format)
|
||||
- [Настройка `date_time_output_format`](../../operations/settings/settings/)
|
||||
- [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
|
||||
- [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime)
|
||||
- [Тип данных `Date`](date.md)
|
||||
- [Тип данных `DateTime64`](datetime64.md)
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/datetime/) <!--hide-->
|
||||
|
||||
@ -92,11 +92,12 @@ FROM dt
|
||||
|
||||
## See Also {#see-also}
|
||||
|
||||
- [Функции преобразования типов](../../sql-reference/data-types/datetime64.md)
|
||||
- [Функции для работы с датой и временем](../../sql-reference/data-types/datetime64.md)
|
||||
- [Функции для работы с массивами](../../sql-reference/data-types/datetime64.md)
|
||||
- [Функции преобразования типов](../../sql-reference/functions/type-conversion-functions.md)
|
||||
- [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md)
|
||||
- [Функции для работы с массивами](../../sql-reference/functions/array-functions.md)
|
||||
- [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format)
|
||||
- [Конфигурационный параметр сервера `timezone`](../../sql-reference/data-types/datetime64.md#server_configuration_parameters-timezone)
|
||||
- [Операторы для работы с датой и временем](../../sql-reference/data-types/datetime64.md#operators-datetime)
|
||||
- [Настройка `date_time_output_format`](../../operations/settings/settings.md)
|
||||
- [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
|
||||
- [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime)
|
||||
- [Тип данных `Date`](date.md)
|
||||
- [Тип данных `DateTime`](datetime.md)
|
||||
|
||||
@ -522,4 +522,55 @@ SELECT * FROM Months WHERE ilike(name, '%j%')
|
||||
!!! note "Примечание"
|
||||
Для случая UTF-8 мы используем триграммное расстояние. Вычисление n-граммного расстояния не совсем честное. Мы используем 2-х байтные хэши для хэширования n-грамм, а затем вычисляем (не)симметрическую разность между хэш таблицами – могут возникнуть коллизии. В формате UTF-8 без учета регистра мы не используем честную функцию `tolower` – мы обнуляем 5-й бит (нумерация с нуля) каждого байта кодовой точки, а также первый бит нулевого байта, если байтов больше 1 – это работает для латиницы и почти для всех кириллических букв.
|
||||
|
||||
## countMatches(haystack, pattern) {#countmatcheshaystack-pattern}
|
||||
|
||||
Возвращает количество совпадений, найденных в строке `haystack`, для регулярного выражения `pattern`.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
countMatches(haystack, pattern)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `haystack` — строка, по которой выполняется поиск. [String](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `pattern` — регулярное выражение, построенное по синтаксическим правилам [re2](https://github.com/google/re2/wiki/Syntax). [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- Количество совпадений.
|
||||
|
||||
Тип: [UInt64](../../sql-reference/data-types/int-uint.md).
|
||||
|
||||
**Примеры**
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT countMatches('foobar.com', 'o+');
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─countMatches('foobar.com', 'o+')─┐
|
||||
│ 2 │
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT countMatches('aaaa', 'aa');
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─countMatches('aaaa', 'aa')────┐
|
||||
│ 2 │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/string_search_functions/) <!--hide-->
|
||||
|
||||
@ -1,58 +1,80 @@
|
||||
# 输入输出格式 {#formats}
|
||||
---
|
||||
toc_priority: 21
|
||||
toc_title: 输入/输出格式
|
||||
---
|
||||
|
||||
ClickHouse 可以接受多种数据格式，可以在 (`INSERT`) 以及 (`SELECT`) 请求中使用。
|
||||
# 输入/输出格式 {#formats}
|
||||
|
||||
下列表格列出了支持的数据格式以及在 (`INSERT`) 以及 (`SELECT`) 请求中使用它们的方式。
|
||||
ClickHouse可以接受和返回各种格式的数据。输入支持的格式可以用来解析提供给`INSERT`的数据，可以从文件备份表(如File, URL或HDFS)执行`SELECT`，或者读取外部字典。输出支持的格式可用于获取`SELECT`的结果，并支持执行`INSERT`文件的表中。
|
||||
|
||||
| 格式 | INSERT | SELECT |
|
||||
|-----------------------------------------------------------------|--------|--------|
|
||||
以下是支持的格式:
|
||||
|
||||
| 格式 | 输入 | 输出 |
|
||||
|-----------------------------------------------------------------------------------------|-------|--------|
|
||||
| [TabSeparated](#tabseparated) | ✔ | ✔ |
|
||||
| [TabSeparatedRaw](#tabseparatedraw) | ✗ | ✔ |
|
||||
| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ |
|
||||
| [模板](#format-template) | ✔ | ✔ |
|
||||
| [Template](#format-template) | ✔ | ✔ |
|
||||
| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ |
|
||||
| [CSV](#csv) | ✔ | ✔ |
|
||||
| [CSVWithNames](#csvwithnames) | ✔ | ✔ |
|
||||
| [自定义分离](#format-customseparated) | ✔ | ✔ |
|
||||
| [值](#data-format-values) | ✔ | ✔ |
|
||||
| [垂直](#vertical) | ✗ | ✔ |
|
||||
| VerticalRaw | ✗ | ✔ |
|
||||
| [CustomSeparated](#format-customseparated) | ✔ | ✔ |
|
||||
| [Values](#data-format-values) | ✔ | ✔ |
|
||||
| [Vertical](#vertical) | ✗ | ✔ |
|
||||
| [VerticalRaw](#verticalraw) | ✗ | ✔ |
|
||||
| [JSON](#json) | ✗ | ✔ |
|
||||
| [JSONAsString](#jsonasstring) | ✔ | ✗ |
|
||||
| [JSONString](#jsonstring) | ✗ | ✔ |
|
||||
| [JSONCompact](#jsoncompact) | ✗ | ✔ |
|
||||
| [JSONCompactString](#jsoncompactstring) | ✗ | ✔ |
|
||||
| [JSONEachRow](#jsoneachrow) | ✔ | ✔ |
|
||||
| [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ |
|
||||
| [JSONStringEachRow](#jsonstringeachrow) | ✔ | ✔ |
|
||||
| [JSONStringEachRowWithProgress](#jsonstringeachrowwithprogress) | ✗ | ✔ |
|
||||
| [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ |
|
||||
| [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ |
|
||||
| [JSONCompactStringEachRow](#jsoncompactstringeachrow) | ✔ | ✔ |
|
||||
| [JSONCompactStringEachRowWithNamesAndTypes](#jsoncompactstringeachrowwithnamesandtypes) | ✔ | ✔ |
|
||||
| [TSKV](#tskv) | ✔ | ✔ |
|
||||
| [漂亮](#pretty) | ✗ | ✔ |
|
||||
| [Pretty](#pretty) | ✗ | ✔ |
|
||||
| [PrettyCompact](#prettycompact) | ✗ | ✔ |
|
||||
| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ |
|
||||
| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ |
|
||||
| [PrettySpace](#prettyspace) | ✗ | ✔ |
|
||||
| [Protobuf](#protobuf) | ✔ | ✔ |
|
||||
| [ProtobufSingle](#protobufsingle) | ✔ | ✔ |
|
||||
| [Avro](#data-format-avro) | ✔ | ✔ |
|
||||
| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ |
|
||||
| [镶木地板](#data-format-parquet) | ✔ | ✔ |
|
||||
| [Parquet](#data-format-parquet) | ✔ | ✔ |
|
||||
| [Arrow](#data-format-arrow) | ✔ | ✔ |
|
||||
| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ |
|
||||
| [ORC](#data-format-orc) | ✔ | ✗ |
|
||||
| [RowBinary](#rowbinary) | ✔ | ✔ |
|
||||
| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ |
|
||||
| [本地人](#native) | ✔ | ✔ |
|
||||
| [Native](#native) | ✔ | ✔ |
|
||||
| [Null](#null) | ✗ | ✔ |
|
||||
| [XML](#xml) | ✗ | ✔ |
|
||||
| [CapnProto](#capnproto) | ✔ | ✔ |
|
||||
| [CapnProto](#capnproto) | ✔ | ✗ |
|
||||
| [LineAsString](#lineasstring) | ✔ | ✗ |
|
||||
|
||||
您可以使用ClickHouse设置控制一些格式处理参数。更多详情设置请参考[设置](../operations/settings/settings.md)
|
||||
|
||||
## TabSeparated {#tabseparated}
|
||||
|
||||
在 TabSeparated 格式中，数据按行写入。每行包含由制表符分隔的值。除了行中的最后一个值（后面紧跟换行符）之外，每个值都跟随一个制表符。 在任何地方都可以使用严格的 Unix 命令行。最后一行还必须在最后包含换行符。值以文本格式编写，不包含引号，并且要转义特殊字符。
|
||||
在TabSeparated分隔格式中，数据按行写入。每行包含由制表符分隔的值。每个值后跟一个制表符，除了行中最后一个值后跟换行。在任何地方都采用严格的Unix换行。最后一行还必须在末尾包含换行。值以文本格式编写，不包含引号，并使用转义的特殊字符。
|
||||
|
||||
这种格式也可以用`TSV`来表示。
|
||||
|
||||
TabSeparated 格式非常方便用于自定义程序或脚本处理数据。HTTP 客户端接口默认会用这种格式，命令行客户端批量模式下也会用这种格式。这种格式允许在不同数据库之间传输数据。例如，从 MYSQL 中导出数据然后导入到 ClickHouse 中，反之亦然。
|
||||
`TabSeparated`格式便于使用自定义程序和脚本处理数据。默认情况下，它在HTTP接口和命令行客户端的批处理模式中使用。这种格式还允许在不同dbms之间传输数据。例如，您可以从MySQL获取转储并将其上传到ClickHouse，反之亦然。
|
||||
|
||||
TabSeparated 格式支持输出数据总值（当使用 WITH TOTALS） 以及极值（当 ‘extremes’ 设置是1）。这种情况下，总值和极值输出在主数据的后面。主要的数据，总值，极值会以一个空行隔开，例如：
|
||||
`TabSeparated`格式支持输出total值(与TOTALS一起使用时)和extreme值(当`extreme`被设置为1时)。在这种情况下，total值和extreme值会在主数据后输出。主要结果、总值和极值之间用空行分隔。示例:
|
||||
|
||||
``` sql
|
||||
SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated``
|
||||
```
|
||||
|
||||
``` text
|
||||
2014-03-17 1406958
|
||||
2014-03-18 1383658
|
||||
2014-03-19 1405797
|
||||
@ -65,115 +87,151 @@ SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORD
|
||||
|
||||
2014-03-17 1031592
|
||||
2014-03-23 1406958
|
||||
```
|
||||
|
||||
### 数据解析方式 {#shu-ju-jie-xi-fang-shi}
|
||||
### 数据格式化 {#data-formatting}
|
||||
|
||||
整数以十进制形式写入。数字在开头可以包含额外的 `+` 字符（解析时忽略，格式化时不记录）。非负数不能包含负号。 读取时，允许将空字符串解析为零，或者（对于带符号的类型）将仅包含负号的字符串解析为零。 不符合相应数据类型的数字可能会被解析为不同的数字，而不会显示错误消息。
|
||||
整数是用十进制形式写的。数字可以在开头包含一个额外的`+`字符(解析时忽略，格式化时不记录)。非负数不能包含负号。在读取时，允许将空字符串解析为零，或者(对于有符号类型)将仅由一个负号组成的字符串解析为零。不符合相应数据类型的数字可以被解析为不同的数字，而不会出现错误消息。
|
||||
|
||||
浮点数以十进制形式写入。点号用作小数点分隔符。支持指数等符号，如’inf’，‘+ inf’，‘-inf’和’nan’。 浮点数的输入可以以小数点开始或结束。
|
||||
格式化的时候，浮点数的精确度可能会丢失。
|
||||
解析的时候，没有严格需要去读取与机器可以表示的最接近的数值。
|
||||
浮点数以十进制形式书写。`.`号用作十进制分隔符。支持指数符号，如`inf`、`+inf`、`-inf`和`nan`。浮点数的条目可以以小数点开始或结束。
|
||||
在格式化期间，浮点数可能会丢失准确性。
|
||||
在解析期间，并不严格要求读取与机器可以表示的最接近的数值。
|
||||
|
||||
日期会以 YYYY-MM-DD 格式写入和解析，但会以任何字符作为分隔符。
|
||||
带时间的日期会以 YYYY-MM-DD hh:mm:ss 格式写入和解析，但会以任何字符作为分隔符。
|
||||
这一切都发生在客户端或服务器启动时的系统时区（取决于哪一种格式的数据）。对于具有时间的日期，夏时制时间未指定。 因此，如果转储在夏令时中有时间，则转储不会明确地匹配数据，解析将选择两者之一。
|
||||
在读取操作期间，不正确的日期和具有时间的日期可以使用自然溢出或空日期和时间进行分析，而不会出现错误消息。
|
||||
日期以YYYY-MM-DD格式编写，并以相同的格式解析，但使用任何字符作为分隔符。
|
||||
日期和时间以`YYYY-MM-DD hh:mm:ss`的格式书写，并以相同的格式解析，但使用任何字符作为分隔符。
|
||||
这一切都发生在客户端或服务器启动时的系统时区(取决于它们对数据的格式)。对于带有时间的日期，夏时制时间未指定。因此，如果转储在夏令时有时间，则转储不会明确地与数据匹配，解析将选择这两次中的一次。
|
||||
在读取操作期间，不正确的日期和具有时间的日期可以使用自然溢出或null日期和时间进行分析，而不会出现错误消息。
|
||||
|
||||
有个例外情况，Unix 时间戳格式（10个十进制数字）也支持使用时间解析日期。结果不是时区相关的。格式 YYYY-MM-DD hh:mm:ss和 NNNNNNNNNN 会自动区分。
|
||||
有个例外情况，Unix时间戳格式也支持用时间解析日期(如果它恰好由10个十进制数字组成)。其结果与时间区域无关。格式`YYYY-MM-DD hh:mm:ss`和`NNNNNNNNNN`是自动区分的。
|
||||
|
||||
字符串以反斜线转义的特殊字符输出。 以下转义序列用于输出：`\b`，`\f`，`\r`，`\n`，`\t`，`\0`，`\'`，`\\`。 解析还支持`\a`，`\v`和`\xHH`（十六进制转义字符）和任何`\c`字符，其中`c`是任何字符（这些序列被转换为`c`）。 因此，读取数据支持可以将换行符写为`\n`或`\`的格式，或者换行。例如，字符串 `Hello world` 在单词之间换行而不是空格可以解析为以下任何形式：
|
||||
字符串以反斜杠转义的特殊字符输出。下面的转义序列用于输出:`\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`。解析还支持`\a`、`\v`和`\xHH`(十六进制转义字符)和任何`\c`字符，其中`c`是任何字符(这些序列被转换为`c`)。因此，读取数据支持这样一种格式，即可以将换行符写成`\n`或`\`，或者写成换行符。例如，字符串`Hello world`在单词之间有换行符，而不是空格，可以用以下语法进行解析:
|
||||
|
||||
``` text
|
||||
Hello\nworld
|
||||
|
||||
Hello\
|
||||
world
|
||||
```
|
||||
|
||||
第二种形式是支持的，因为MySQL读取tab-separated格式数据集的时候也会使用它。
|
||||
|
||||
在 TabSeparated 格式中传递数据时需要转义的最小字符集为：Tab，换行符（LF）和反斜杠。
|
||||
在TabSeparated分隔格式传递数据时需要转义的最小字符集:`Tab`、换行符(LF)和反斜杠。
|
||||
|
||||
只有一小组符号会被转义。你可以轻易地找到一个字符串值，但这不会正常在你的终端显示。
|
||||
只有一小部分符号被转义。您可以很容易地找到一个字符串值，而您的终端将在输出中不显示它。
|
||||
|
||||
数组写在方括号内的逗号分隔值列表中。 通常情况下，数组中的数字项目会被拼凑，但日期，带时间的日期以及字符串将使用与上面相同的转义规则用单引号引起来。
|
||||
数组写在方括号内的逗号分隔值列表中。数组中的数字项按正常格式进行格式化。`Date`和`DateTime`类型用单引号表示。字符串使用与上面相同的转义规则在单引号中编写。
|
||||
|
||||
[NULL](../sql-reference/syntax.md)将输出为`\N`。
|
||||
|
||||
[Nested](../sql-reference/data-types/nested-data-structures/nested.md)结构的每个元素都表示为数组。
|
||||
|
||||
示例：
|
||||
|
||||
``` sql
|
||||
CREATE TABLE nestedt
|
||||
(
|
||||
`id` UInt8,
|
||||
`aux` Nested(
|
||||
a UInt8,
|
||||
b String
|
||||
)
|
||||
)
|
||||
ENGINE = TinyLog
|
||||
```
|
||||
|
||||
``` sql
|
||||
INSERT INTO nestedt Values ( 1, [1], ['a'])
|
||||
```
|
||||
|
||||
``` sql
|
||||
SELECT * FROM nestedt FORMAT TSV
|
||||
```
|
||||
|
||||
``` text
|
||||
1 [1] ['a']
|
||||
```
|
||||
|
||||
## TabSeparatedRaw {#tabseparatedraw}
|
||||
|
||||
与 `TabSeparated` 格式不一样的是，行数据是不会被转义的。
|
||||
该格式仅适用于输出查询结果，但不适用于解析输入（将数据插入到表中）。
|
||||
与`TabSeparated`格式的不同之处在于，写入的行没有转义。
|
||||
使用这种格式解析时，每个字段中不允许使用制表符或换行符。
|
||||
|
||||
这种格式也可以使用名称`TSVRaw`来表示。
|
||||
|
||||
## TabSeparatedWithNames {#tabseparatedwithnames}
|
||||
|
||||
与 `TabSeparated` 格式不一样的是，第一行会显示列的名称。
|
||||
在解析过程中，第一行完全被忽略。您不能使用列名来确定其位置或检查其正确性。
|
||||
（未来可能会加入解析头行的功能）
|
||||
与`TabSeparated`格式不同的是列名写在第一行。
|
||||
在解析过程中，第一行被完全忽略。不能使用列名来确定它们的位置或检查它们的正确性。
|
||||
(将来可能会添加对头行解析的支持。)
|
||||
|
||||
这种格式也可以使用名称`TSVWithNames`来表示。
|
||||
|
||||
## TabSeparatedWithNamesAndTypes {#tabseparatedwithnamesandtypes}
|
||||
|
||||
与 `TabSeparated` 格式不一样的是，第一行会显示列的名称，第二行会显示列的类型。
|
||||
在解析过程中，第一行和第二行完全被忽略。
|
||||
与`TabSeparated`格式不同的是列名写在第一行，而列类型写在第二行。
|
||||
在解析过程中，将完全忽略第一行和第二行。
|
||||
|
||||
这种格式也可以使用名称`TSVWithNamesAndTypes`来表示。
|
||||
|
||||
## 模板 {#format-template}
|
||||
## Template {#format-template}
|
||||
|
||||
此格式允许为具有指定转义规则的值指定带有占位符的自定义格式字符串。
|
||||
此格式允许指定带有占位符的自定义格式字符串，这些占位符用于指定转义规则。
|
||||
|
||||
它使用设置 `format_schema`, `format_schema_rows`, `format_schema_rows_between_delimiter` and some settings of other formats (e.g. `output_format_json_quote_64bit_integers` 使用时 `JSON` 逃跑，进一步查看)
|
||||
它使用设置`format_schema`, `format_schema_rows`, `format_schema_rows_between_delimiter`以及其他格式的一些设置(例如转义`JSON`时使用`output_format_json_quote_64bit_integers`)
|
||||
|
||||
格式字符串 `format_schema_rows` 使用以下语法指定行格式:
|
||||
设置`format_template_row`指定文件的路径，该文件包含以下语法的行格式字符串:
|
||||
|
||||
`delimiter_1${column_1:serializeAs_1}delimiter_2${column_2:serializeAs_2} ... delimiter_N`,
|
||||
|
||||
where `delimiter_i` is a delimiter between values (`$` symbol can be escaped as `$$`),
|
||||
`column_i` is a name of a column whose values are to be selected or inserted (if empty, then column will be skipped),
|
||||
`serializeAs_i` is an escaping rule for the column values. The following escaping rules are supported:
|
||||
其中，`delimiter_i`是值之间的分隔符(`$`符号可以转义为`$$`)，
|
||||
`column_i`是要选择或插入其值的列的名称或索引(如果为空，则跳过该列)，
|
||||
`serializeAs_i`是列值的转义规则。支持以下转义规则:
|
||||
|
||||
- `CSV`, `JSON`, `XML` (similarly to the formats of the same names)
|
||||
- `Escaped` (similarly to `TSV`)
|
||||
- `Quoted` (similarly to `Values`)
|
||||
- `Raw` (without escaping, similarly to `TSVRaw`)
|
||||
- `None` (no escaping rule, see further)
|
||||
- `CSV`, `JSON`, `XML` (类似于相同名称的格式)
|
||||
- `Escaped` (类似于`TSV`)
|
||||
- `Quoted` (类似于`Values`)
|
||||
- `Raw` (类似于`TSVRaw`)
|
||||
- `None`
|
||||
|
||||
If escaping rule is omitted, then`None` will be used. `XML` and `Raw` are suitable only for output.
|
||||
如果省略了转义规则，那么将使用`None`。`XML`和`Raw`只适用于输出。
|
||||
|
||||
So, for the following format string:
|
||||
对于下面的格式字符串:
|
||||
|
||||
`Search phrase: ${SearchPhrase:Quoted}, count: ${c:Escaped}, ad price: $$${price:JSON};`
|
||||
|
||||
the values of `SearchPhrase`, `c` and `price` columns, which are escaped as `Quoted`, `Escaped` and `JSON` will be printed (for select) or will be expected (for insert) between `Search phrase: `, `, count: `, `, ad price: $` and `;` delimiters respectively. For example:
|
||||
`SearchPhrase`、`c`和`price`列的值被转义为`quotation`、`Escaped`和`JSON`将分别在`Search phrase:`， `， count: `， `， ad price: $`和`;`分隔符之间打印(用于选择)或expected(用于插入)。例如:
|
||||
|
||||
`Search phrase: 'bathroom interior design', count: 2166, ad price: $3;`
|
||||
|
||||
该 `format_schema_rows_between_delimiter` setting指定行之间的分隔符，该分隔符在除最后一行之外的每一行之后打印（或预期） (`\n` 默认情况下)
|
||||
`format_template_rows_between_delimiter`设置指定行之间的分隔符，它将打印(或expected)在每一行之后，最后一行除外(默认为`\n`)。
|
||||
|
||||
格式字符串 `format_schema` 具有相同的语法 `format_schema_rows` 并允许指定前缀，后缀和打印一些附加信息的方式。 它包含以下占位符而不是列名:
|
||||
设置`format_template_resultset`指定文件路径，该文件包含resultset的格式字符串。resultset的格式字符串与row的格式字符串具有相同的语法，允许指定前缀、后缀和打印一些附加信息的方法。它包含以下占位符而不是列名:
|
||||
|
||||
- `data` 包含数据的行 `format_schema_rows` 格式，由分隔 `format_schema_rows_between_delimiter`. 此占位符必须是格式字符串中的第一个占位符。
|
||||
- `totals` 是包含总值的行 `format_schema_rows` 格式（与总计一起使用时)
|
||||
- `min` 是具有最小值的行 `format_schema_rows` 格式（当极值设置为1时)
|
||||
- `max` 是具有最大值的行 `format_schema_rows` 格式（当极值设置为1时)
|
||||
- `rows` 输出行总数
|
||||
- `rows_before_limit` 是没有限制的最小行数。 仅当查询包含LIMIT时输出。 如果查询包含GROUP BY，则rows_before_limit_at_least是没有限制的确切行数。
|
||||
- `time` 请求执行时间以秒为单位
|
||||
- `data` `format_template_row`格式的数据行，由`format_template_rows_between_delimiter`分隔。此占位符必须是格式字符串中的第一个占位符。
|
||||
- `totals` `format_template_row`格式的总值(和WITH TOTALS一起使用)
|
||||
- `min` `format_template_row`格式的最小值(当极值设置为1时)
|
||||
- `max` `format_template_row`格式的最大值(当极值设置为1时)
|
||||
- `rows` 输出行的总数
|
||||
- `rows_before_limit` 没有LIMIT的最小行数。仅当查询包含LIMIT时输出。如果查询包含GROUP BY，那么rows_before_limit_at_least就是没有LIMIT的确切行数。
|
||||
- `time` 请求执行时间（秒）
|
||||
- `rows_read` 已读取的行数
|
||||
- `bytes_read` 被读取的字节数（未压缩）
|
||||
- `bytes_read` 已读取（未压缩）的字节数
|
||||
|
||||
占位符 `data`, `totals`, `min` 和 `max` 必须没有指定转义规则（或 `None` 必须明确指定）。 其余的占位符可能具有指定的任何转义规则。
|
||||
如果 `format_schema` 设置为空字符串, `${data}` 用作默认值。
|
||||
对于插入查询格式允许跳过一些列或一些字段，如果前缀或后缀（见示例）。
|
||||
占位符`data`、`totals`、`min`和`max`必须没有指定转义规则(或者必须显式指定`None`)。其余占位符可以指定任何转义规则。
|
||||
如果`format_template_resultset`设置为空字符串，则使用`${data}`作为默认值。
|
||||
对于insert查询，格式允许跳过某些列或某些字段的前缀或后缀(参见示例)。
|
||||
|
||||
`Select` 示例:
|
||||
Select示例:
|
||||
|
||||
``` sql
|
||||
SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase ORDER BY c DESC LIMIT 5
|
||||
FORMAT Template
|
||||
SETTINGS format_schema = '<!DOCTYPE HTML>
|
||||
SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase ORDER BY c DESC LIMIT 5 FORMAT Template SETTINGS
|
||||
format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format', format_template_rows_between_delimiter = '\n '
|
||||
```
|
||||
|
||||
`/some/path/resultset.format`:
|
||||
|
||||
``` text
|
||||
<!DOCTYPE HTML>
|
||||
<html> <head> <title>Search phrases</title> </head>
|
||||
<body>
|
||||
<table border="1"> <caption>Search phrases</caption>
|
||||
@ -185,11 +243,17 @@ SETTINGS format_schema = '<!DOCTYPE HTML>
|
||||
</table>
|
||||
<b>Processed ${rows_read:XML} rows in ${time:XML} sec</b>
|
||||
</body>
|
||||
</html>',
|
||||
format_schema_rows = '<tr> <td>${SearchPhrase:XML}</td> <td>${с:XML}</td> </tr>',
|
||||
format_schema_rows_between_delimiter = '\n '
|
||||
</html>
|
||||
```
|
||||
|
||||
`/some/path/row.format`:
|
||||
|
||||
``` text
|
||||
<tr> <td>${0:XML}</td> <td>${1:XML}</td> </tr>
|
||||
```
|
||||
|
||||
结果：
|
||||
|
||||
``` html
|
||||
<!DOCTYPE HTML>
|
||||
<html> <head> <title>Search phrases</title> </head>
|
||||
@ -210,39 +274,63 @@ format_schema_rows_between_delimiter = '\n '
|
||||
</html>
|
||||
```
|
||||
|
||||
`Insert` 示例:
|
||||
Insert示例：
|
||||
|
||||
``` text
|
||||
Some header
|
||||
Page views: 5, User id: 4324182021466249494, Useless field: hello, Duration: 146, Sign: -1
|
||||
Page views: 6, User id: 4324182021466249494, Useless field: world, Duration: 185, Sign: 1
|
||||
Total rows: 2
|
||||
```
|
||||
|
||||
``` sql
|
||||
INSERT INTO UserActivity FORMAT Template SETTINGS
|
||||
format_schema = 'Some header\n${data}\nTotal rows: ${:CSV}\n',
|
||||
format_schema_rows = 'Page views: ${PageViews:CSV}, User id: ${UserID:CSV}, Useless field: ${:CSV}, Duration: ${Duration:CSV}, Sign: ${Sign:CSV}'
|
||||
format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format'
|
||||
```
|
||||
|
||||
`PageViews`, `UserID`, `Duration` 和 `Sign` 占位符内部是表中列的名称。 值后 `Useless field` 在行和之后 `\nTotal rows:` in后缀将被忽略。
|
||||
`/some/path/resultset.format`:
|
||||
|
||||
``` text
|
||||
Some header\n${data}\nTotal rows: ${:CSV}\n
|
||||
```
|
||||
|
||||
`/some/path/row.format`:
|
||||
|
||||
``` text
|
||||
Page views: ${PageViews:CSV}, User id: ${UserID:CSV}, Useless field: ${:CSV}, Duration: ${Duration:CSV}, Sign: ${Sign:CSV}
|
||||
```
|
||||
|
||||
`PageViews`, `UserID`, `Duration`和`Sign` 内部占位符是表中列的名称。将忽略行中`Useless field`后面和后缀中`\nTotal rows:`之后的值。
|
||||
输入数据中的所有分隔符必须严格等于指定格式字符串中的分隔符。
|
||||
|
||||
## TemplateIgnoreSpaces {#templateignorespaces}
|
||||
|
||||
此格式仅适用于输入。
|
||||
类似于 `Template`，但跳过输入流中的分隔符和值之间的空格字符。 但是，如果格式字符串包含空格字符，则在输入流中将需要这些字符。 还允许指定空占位符 (`${}` 或 `${:None}`）将一些分隔符分成单独的部分，以忽略它们之间的空格。 此类占位符仅用于跳过空格字符。
|
||||
可以阅读 `JSON` 如果列的值在所有行中具有相同的顺序，则使用此格式。 例如，以下请求可用于从格式的输出示例中插入数据 [JSON](#json):
|
||||
这种格式只适用于输入。
|
||||
类似于`Template`，但跳过输入流中分隔符和值之间的空白字符。但是，如果格式字符串包含空格字符，这些字符将会出现在输入流中。还允许指定空占位符(`${}`或`${:None}`)来将一些分隔符分割为单独的部分，以忽略它们之间的空格。这种占位符仅用于跳过空白字符。
|
||||
如果列的值在所有行的顺序相同，那么可以使用这种格式读取`JSON`。可以使用以下请求从格式为[JSON](#json)的输出示例中插入数据：
|
||||
|
||||
``` sql
|
||||
INSERT INTO table_name FORMAT TemplateIgnoreSpaces SETTINGS
|
||||
format_schema = '{${}"meta"${}:${:JSON},${}"data"${}:${}[${data}]${},${}"totals"${}:${:JSON},${}"extremes"${}:${:JSON},${}"rows"${}:${:JSON},${}"rows_before_limit_at_least"${}:${:JSON}${}}',
|
||||
format_schema_rows = '{${}"SearchPhrase"${}:${}${phrase:JSON}${},${}"c"${}:${}${cnt:JSON}${}}',
|
||||
format_schema_rows_between_delimiter = ','
|
||||
format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format', format_template_rows_between_delimiter = ','
|
||||
```
|
||||
|
||||
`/some/path/resultset.format`:
|
||||
|
||||
``` text
|
||||
{${}"meta"${}:${:JSON},${}"data"${}:${}[${data}]${},${}"totals"${}:${:JSON},${}"extremes"${}:${:JSON},${}"rows"${}:${:JSON},${}"rows_before_limit_at_least"${}:${:JSON}${}}
|
||||
```
|
||||
|
||||
`/some/path/row.format`:
|
||||
|
||||
``` text
|
||||
{${}"SearchPhrase"${}:${}${phrase:JSON}${},${}"c"${}:${}${cnt:JSON}${}}
|
||||
```
|
||||
|
||||
## TSKV {#tskv}
|
||||
|
||||
与 `TabSeparated` 格式类似，但它输出的是 `name=value` 的格式。名称会和 `TabSeparated` 格式一样被转义，`=` 字符也会被转义。
|
||||
类似于TabSeparated，但是输出的值是name=value格式。名称的转义方式与TabSeparated格式相同，=符号也是转义的。
|
||||
|
||||
``` text
|
||||
SearchPhrase= count()=8267016
|
||||
SearchPhrase=bathroom interior design count()=2166
|
||||
SearchPhrase=yandex count()=1655
|
||||
@ -253,49 +341,55 @@ format_schema_rows_between_delimiter = ','
|
||||
SearchPhrase=photos of dog breeds count()=1091
|
||||
SearchPhrase=curtain designs count()=1064
|
||||
SearchPhrase=baku count()=1000
|
||||
```
|
||||
|
||||
[NULL](../sql-reference/syntax.md) 输出为 `\N`。
|
||||
[NULL](../sql-reference/syntax.md)格式为`\N`。
|
||||
|
||||
``` sql
|
||||
SELECT * FROM t_null FORMAT TSKV
|
||||
```
|
||||
|
||||
``` text
|
||||
x=1 y=\N
|
||||
```
|
||||
|
||||
当有大量的小列时，这种格式是低效的，通常没有理由使用它。它被用于 Yandex 公司的一些部门。
|
||||
当有大量的小列时，这种格式是无效的，并且通常没有理由使用它。不过，就效率而言，它并不比JSONEachRow差。
|
||||
这种格式支持数据输出和解析。对于解析，不同列的值支持任何顺序。省略某些值是可以接受的——它们被视为与其默认值相等。在这种情况下，0和空白行被用作默认值。不支持在表中指定的复杂值作为缺省值。
|
||||
|
||||
数据的输出和解析都支持这种格式。对于解析，任何顺序都支持不同列的值。可以省略某些值，用 `-` 表示， 它们被视为等于它们的默认值。在这种情况下，零和空行被用作默认值。作为默认值，不支持表中指定的复杂值。
|
||||
|
||||
对于不带等号或值，可以用附加字段 `tskv` 来表示，这种在解析上是被允许的。这样的话该字段被忽略。
|
||||
解析允许存在不带等号或值的附加字段`tskv`。此字段被忽略。
|
||||
|
||||
## CSV {#csv}
|
||||
|
||||
按逗号分隔的数据格式([RFC](https://tools.ietf.org/html/rfc4180))。
|
||||
按`,`分隔的数据格式([RFC](https://tools.ietf.org/html/rfc4180))。
|
||||
|
||||
格式化的时候，行是用双引号括起来的。字符串中的双引号会以两个双引号输出，除此之外没有其他规则来做字符转义了。日期和时间也会以双引号包括。数字的输出不带引号。值由一个单独的字符隔开，这个字符默认是 `,`。行使用 Unix 换行符（LF）分隔。 数组序列化成 CSV 规则如下：首先将数组序列化为 TabSeparated 格式的字符串，然后将结果字符串用双引号包括输出到 CSV。CSV 格式的元组被序列化为单独的列（即它们在元组中的嵌套关系会丢失）。
|
||||
格式化时，行是用双引号括起来的。字符串中的双引号会以两个双引号输出，除此之外没有其他规则来做字符转义了。日期和时间也会以双引号包括。数字的输出不带引号。值由一个单独的字符隔开，这个字符默认是`,`。行使用Unix换行符（LF）分隔。数组序列化成CSV规则如下：首先将数组序列化为`TabSeparated`格式的字符串，然后将结果字符串用双引号包括输出到`CSV`。`CSV`格式的元组被序列化为单独的列（即它们在元组中的嵌套关系会丢失）。
|
||||
|
||||
clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv
|
||||
``` bash
|
||||
$ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv
|
||||
```
|
||||
|
||||
\*默认情况下间隔符是 `,` ，在 [format_csv_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter) 中可以了解更多间隔符配置。
|
||||
\* 默认情况下间隔符是`,` ，在[format_csv_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter)中可以了解更多分隔符配置。
|
||||
|
||||
解析的时候，可以使用或不使用引号来解析所有值。支持双引号和单引号。行也可以不用引号排列。 在这种情况下，它们被解析为逗号或换行符（CR 或 LF）。在解析不带引号的行时，若违反 RFC 规则，会忽略前导和尾随的空格和制表符。 对于换行，全部支持 Unix（LF），Windows（CR LF）和 Mac OS Classic（CR LF）。
|
||||
解析的时候，可以使用或不使用引号来解析所有值。支持双引号和单引号。行也可以不用引号排列。在这种情况下，它们被解析为逗号或换行符（`CR或`LF`）。在解析不带引号的行时，若违反`RFC`规则，会忽略前缀和结尾的空格和制表符。对于换行，全部支持Unix（LF），Windows（CR LF）和Mac OS Classic（CR LF）。
|
||||
|
||||
`NULL` 将输出为 `\N`。
|
||||
如果启用[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)，空的末尾加引号的输入值将替换为相应列的默认值。
|
||||
|
||||
CSV 格式是和 TabSeparated 一样的方式输出总数和极值。
|
||||
`NULL`被格式化为`\N`或`NULL`或一个空的非引号字符串(详见配置[input_format_csv_unquoted_null_literal_as_null](../operations/settings/settings.md#settings-input_format_csv_unquoted_null_literal_as_null)或[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields))。
|
||||
|
||||
`CSV`格式支持输出总数和极值的方式与`TabSeparated`相同。
|
||||
|
||||
## CSVWithNames {#csvwithnames}
|
||||
|
||||
会输出带头部行，和 `TabSeparatedWithNames` 一样。
|
||||
会输出带头部的信息(字段列表)，和`TabSeparatedWithNames`一样。
|
||||
|
||||
## 自定义分离 {#format-customseparated}
|
||||
## CustomSeparated {#format-customseparated}
|
||||
|
||||
类似于 [模板](#format-template)，但它打印或读取所有列，并使用从设置转义规则 `format_custom_escaping_rule` 从设置和分隔符 `format_custom_field_delimiter`, `format_custom_row_before_delimiter`, `format_custom_row_after_delimiter`, `format_custom_row_between_delimiter`, `format_custom_result_before_delimiter` 和 `format_custom_result_after_delimiter`，而不是从格式字符串。
|
||||
也有 `CustomSeparatedIgnoreSpaces` 格式，这是类似于 `TemplateIgnoreSpaces`.
|
||||
类似于[Template](#format-template)， 但它打印或读取所有列和使用转义规则在设置`format_custom_escaping_rule`和分隔符设置`format_custom_field_delimiter`,`format_custom_row_before_delimiter`,`format_custom_row_after_delimiter`,`format_custom_row_between_delimiter`,`format_custom_result_before_delimiter`,`format_custom_result_after_delimiter`中,而不是从格式字符串。
|
||||
也有`CustomSeparatedIgnoreSpaces`格式，这是类似于`TemplateIgnoreSpaces`。
|
||||
|
||||
## JSON {#json}
|
||||
|
||||
以 JSON 格式输出数据。除了数据表之外，它还输出列名称和类型以及一些附加信息：输出行的总数以及在没有 LIMIT 时可以输出的行数。 例：
|
||||
以JSON格式输出数据。除了数据表之外，它还输出列名和类型，以及一些附加信息: 输出行的总数，以及如果没有LIMIT的话可输出的行数。示例:
|
||||
|
||||
``` sql
|
||||
SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTALS ORDER BY c DESC LIMIT 5 FORMAT JSON
|
||||
@ -306,71 +400,50 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA
|
||||
"meta":
|
||||
[
|
||||
{
|
||||
"name": "SearchPhrase",
|
||||
"name": "'hello'",
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"name": "c",
|
||||
"name": "multiply(42, number)",
|
||||
"type": "UInt64"
|
||||
},
|
||||
{
|
||||
"name": "range(5)",
|
||||
"type": "Array(UInt8)"
|
||||
}
|
||||
],
|
||||
|
||||
"data":
|
||||
[
|
||||
{
|
||||
"SearchPhrase": "",
|
||||
"c": "8267016"
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "0",
|
||||
"range(5)": [0,1,2,3,4]
|
||||
},
|
||||
{
|
||||
"SearchPhrase": "bathroom interior design",
|
||||
"c": "2166"
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "42",
|
||||
"range(5)": [0,1,2,3,4]
|
||||
},
|
||||
{
|
||||
"SearchPhrase": "yandex",
|
||||
"c": "1655"
|
||||
},
|
||||
{
|
||||
"SearchPhrase": "spring 2014 fashion",
|
||||
"c": "1549"
|
||||
},
|
||||
{
|
||||
"SearchPhrase": "freeform photos",
|
||||
"c": "1480"
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "84",
|
||||
"range(5)": [0,1,2,3,4]
|
||||
}
|
||||
],
|
||||
|
||||
"totals":
|
||||
{
|
||||
"SearchPhrase": "",
|
||||
"c": "8873898"
|
||||
},
|
||||
"rows": 3,
|
||||
|
||||
"extremes":
|
||||
{
|
||||
"min":
|
||||
{
|
||||
"SearchPhrase": "",
|
||||
"c": "1480"
|
||||
},
|
||||
"max":
|
||||
{
|
||||
"SearchPhrase": "",
|
||||
"c": "8267016"
|
||||
}
|
||||
},
|
||||
|
||||
"rows": 5,
|
||||
|
||||
"rows_before_limit_at_least": 141137
|
||||
"rows_before_limit_at_least": 3
|
||||
}
|
||||
```
|
||||
|
||||
JSON 与 JavaScript 兼容。为了确保这一点，一些字符被另外转义：斜线`/`被转义为`\/`; 替代的换行符 `U+2028` 和 `U+2029` 会打断一些浏览器解析，它们会被转义为 `\uXXXX`。 ASCII 控制字符被转义：退格，换页，换行，回车和水平制表符被替换为`\b`，`\f`，`\n`，`\r`，`\t` 作为使用`\uXXXX`序列的00-1F范围内的剩余字节。 无效的 UTF-8 序列更改为替换字符 ，因此输出文本将包含有效的 UTF-8 序列。 为了与 JavaScript 兼容，默认情况下，Int64 和 UInt64 整数用双引号引起来。要除去引号，可以将配置参数 output_format_json_quote_64bit_integers 设置为0。
|
||||
JSON与JavaScript兼容。为了确保这一点，一些字符被另外转义：斜线`/`被转义为`\/`; 替代的换行符`U+2028`和`U+2029`会打断一些浏览器解析，它们会被转义为`\uXXXX`。 ASCII控制字符被转义：退格，换页，换行，回车和水平制表符被替换为`\b`，`\f`，`\n`，`\r`，`\t` 作为使用`\uXXXX`序列的00-1F范围内的剩余字节。 无效的UTF-8序列更改为替换字符，因此输出文本将包含有效的UTF-8序列。 为了与JavaScript兼容，默认情况下，Int64和UInt64整数用双引号引起来。要除去引号，可以将配置参数`output_format_json_quote_64bit_integers`设置为0。
|
||||
|
||||
`rows` – 结果输出的行数。
|
||||
|
||||
`rows_before_limit_at_least`去掉 LIMIT过滤后的最小行总数。 只会在查询包含LIMIT条件时输出。
|
||||
若查询包含 GROUP BY，rows_before_limit_at_least 就是去掉 LIMIT 后过滤后的准确行数。
|
||||
若查询包含 GROUP BY，`rows_before_limit_at_least`就是去掉LIMIT后过滤后的准确行数。
|
||||
|
||||
`totals` – 总值 （当使用TOTALS条件时）。
|
||||
|
||||
@ -378,13 +451,16 @@ JSON 与 JavaScript 兼容。为了确保这一点，一些字符被另外转义
|
||||
|
||||
该格式仅适用于输出查询结果，但不适用于解析输入（将数据插入到表中）。
|
||||
|
||||
ClickHouse 支持 [NULL](../sql-reference/syntax.md), 在 JSON 格式中以 `null` 输出来表示.
|
||||
ClickHouse支持[NULL](../sql-reference/syntax.md), 在JSON输出中显示为`null`。若要在输出中启用`+nan`、`-nan`、`+inf`、`-inf`值，请设置[output_format_json_quote_denormals](../operations/settings/settings.md#settings-output_format_json_quote_denormals)为1。
|
||||
|
||||
参考 JSONEachRow 格式。
|
||||
**参考**
|
||||
|
||||
## JSONCompact {#jsoncompact}
|
||||
- [JSONEachRow](#jsoneachrow)格式
|
||||
- [output_format_json_array_of_rows](../operations/settings/settings.md#output-format-json-array-of-rows)设置
|
||||
|
||||
与 JSON 格式不同的是它以数组的方式输出结果，而不是以结构体。
|
||||
## JSONString {#jsonstring}
|
||||
|
||||
与JSON的不同之处在于数据字段以字符串输出，而不是以类型化JSON值输出。
|
||||
|
||||
示例：
|
||||
|
||||
@ -393,79 +469,257 @@ ClickHouse 支持 [NULL](../sql-reference/syntax.md), 在 JSON 格式中以 `nul
|
||||
"meta":
|
||||
[
|
||||
{
|
||||
"name": "SearchPhrase",
|
||||
"name": "'hello'",
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"name": "c",
|
||||
"name": "multiply(42, number)",
|
||||
"type": "UInt64"
|
||||
},
|
||||
{
|
||||
"name": "range(5)",
|
||||
"type": "Array(UInt8)"
|
||||
}
|
||||
],
|
||||
|
||||
"data":
|
||||
[
|
||||
["", "8267016"],
|
||||
["bathroom interior design", "2166"],
|
||||
["yandex", "1655"],
|
||||
["fashion trends spring 2014", "1549"],
|
||||
["freeform photo", "1480"]
|
||||
{
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "0",
|
||||
"range(5)": "[0,1,2,3,4]"
|
||||
},
|
||||
{
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "42",
|
||||
"range(5)": "[0,1,2,3,4]"
|
||||
},
|
||||
{
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "84",
|
||||
"range(5)": "[0,1,2,3,4]"
|
||||
}
|
||||
],
|
||||
|
||||
"totals": ["","8873898"],
|
||||
"rows": 3,
|
||||
|
||||
"extremes":
|
||||
{
|
||||
"min": ["","1480"],
|
||||
"max": ["","8267016"]
|
||||
},
|
||||
|
||||
"rows": 5,
|
||||
|
||||
"rows_before_limit_at_least": 141137
|
||||
"rows_before_limit_at_least": 3
|
||||
}
|
||||
```
|
||||
|
||||
这种格式仅仅适用于输出结果集，而不适用于解析（将数据插入到表中）。
|
||||
参考 `JSONEachRow` 格式。
|
||||
## JSONAsString {#jsonasstring}
|
||||
|
||||
## JSONEachRow {#jsoneachrow}
|
||||
在这种格式中，一个JSON对象被解释为一个值。如果输入有几个JSON对象(逗号分隔)，它们将被解释为独立的行。
|
||||
|
||||
将数据结果每一行以 JSON 结构体输出（换行分割 JSON 结构体）。
|
||||
这种格式只能对具有单个字段类型的表进行解析[String](../sql-reference/data-types/string.md)。其余的列必须设置为[DEFAULT](../sql-reference/statements/create.md)或[MATERIALIZED](../sql-reference/statements/create.md)，或者忽略。一旦将整个JSON对象收集为字符串，就可以使用[JSON函数](../sql-reference/functions/json-functions.md)运行它。
|
||||
|
||||
``` json
|
||||
{"SearchPhrase":"","count()":"8267016"}
|
||||
{"SearchPhrase": "bathroom interior design","count()": "2166"}
|
||||
{"SearchPhrase":"yandex","count()":"1655"}
|
||||
{"SearchPhrase":"2014 spring fashion","count()":"1549"}
|
||||
{"SearchPhrase":"freeform photo","count()":"1480"}
|
||||
{"SearchPhrase":"angelina jolie","count()":"1245"}
|
||||
{"SearchPhrase":"omsk","count()":"1112"}
|
||||
{"SearchPhrase":"photos of dog breeds","count()":"1091"}
|
||||
{"SearchPhrase":"curtain designs","count()":"1064"}
|
||||
{"SearchPhrase":"baku","count()":"1000"}
|
||||
**示例**
|
||||
|
||||
查询：
|
||||
|
||||
``` sql
|
||||
DROP TABLE IF EXISTS json_as_string;
|
||||
CREATE TABLE json_as_string (json String) ENGINE = Memory;
|
||||
INSERT INTO json_as_string FORMAT JSONAsString {"foo":{"bar":{"x":"y"},"baz":1}},{},{"any json stucture":1}
|
||||
SELECT * FROM json_as_string;
|
||||
```
|
||||
|
||||
与 JSON 格式不同的是，没有替换无效的UTF-8序列。任何一组字节都可以在行中输出。这是必要的，因为这样数据可以被格式化而不会丢失任何信息。值的转义方式与JSON相同。
|
||||
结果：
|
||||
|
||||
对于解析，任何顺序都支持不同列的值。可以省略某些值 - 它们被视为等于它们的默认值。在这种情况下，零和空行被用作默认值。 作为默认值，不支持表中指定的复杂值。元素之间的空白字符被忽略。如果在对象之后放置逗号，它将被忽略。对象不一定必须用新行分隔。
|
||||
``` text
|
||||
┌─json──────────────────────────────┐
|
||||
│ {"foo":{"bar":{"x":"y"},"baz":1}} │
|
||||
│ {} │
|
||||
│ {"any json stucture":1} │
|
||||
└───────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 嵌套结构的使用 {#jsoneachrow-nested}
|
||||
## JSONCompact {#jsoncompact}
|
||||
## JSONCompactString {#jsoncompactstring}
|
||||
|
||||
如果你有一张桌子 [嵌套式](../sql-reference/data-types/nested-data-structures/nested.md) 数据类型列，可以插入具有相同结构的JSON数据。 启用此功能与 [input_format_import_nested_json](../operations/settings/settings.md#settings-input_format_import_nested_json) 设置。
|
||||
与JSON格式不同的是它以数组的方式输出结果，而不是以结构体。
|
||||
|
||||
例如，请考虑下表:
|
||||
示例：
|
||||
|
||||
``` json
|
||||
// JSONCompact
|
||||
{
|
||||
"meta":
|
||||
[
|
||||
{
|
||||
"name": "'hello'",
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"name": "multiply(42, number)",
|
||||
"type": "UInt64"
|
||||
},
|
||||
{
|
||||
"name": "range(5)",
|
||||
"type": "Array(UInt8)"
|
||||
}
|
||||
],
|
||||
|
||||
"data":
|
||||
[
|
||||
["hello", "0", [0,1,2,3,4]],
|
||||
["hello", "42", [0,1,2,3,4]],
|
||||
["hello", "84", [0,1,2,3,4]]
|
||||
],
|
||||
|
||||
"rows": 3,
|
||||
|
||||
"rows_before_limit_at_least": 3
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
// JSONCompactString
|
||||
{
|
||||
"meta":
|
||||
[
|
||||
{
|
||||
"name": "'hello'",
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"name": "multiply(42, number)",
|
||||
"type": "UInt64"
|
||||
},
|
||||
{
|
||||
"name": "range(5)",
|
||||
"type": "Array(UInt8)"
|
||||
}
|
||||
],
|
||||
|
||||
"data":
|
||||
[
|
||||
["hello", "0", "[0,1,2,3,4]"],
|
||||
["hello", "42", "[0,1,2,3,4]"],
|
||||
["hello", "84", "[0,1,2,3,4]"]
|
||||
],
|
||||
|
||||
"rows": 3,
|
||||
|
||||
"rows_before_limit_at_least": 3
|
||||
}
|
||||
```
|
||||
|
||||
## JSONEachRow {#jsoneachrow}
|
||||
## JSONStringEachRow {#jsonstringeachrow}
|
||||
## JSONCompactEachRow {#jsoncompacteachrow}
|
||||
## JSONCompactStringEachRow {#jsoncompactstringeachrow}
|
||||
|
||||
使用这些格式时，ClickHouse会将行输出为分隔的、换行分隔的JSON值，但数据作为一个整体不是有效的JSON。
|
||||
|
||||
``` json
|
||||
{"some_int":42,"some_str":"hello","some_tuple":[1,"a"]} // JSONEachRow
|
||||
[42,"hello",[1,"a"]] // JSONCompactEachRow
|
||||
["42","hello","(2,'a')"] // JSONCompactStringsEachRow
|
||||
```
|
||||
|
||||
在插入数据时，应该为每一行提供一个单独的JSON值。
|
||||
|
||||
## JSONEachRowWithProgress {#jsoneachrowwithprogress}
|
||||
## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress}
|
||||
|
||||
与`JSONEachRow`/`JSONStringEachRow`不同的是，ClickHouse还将生成作为JSON值的进度信息。
|
||||
|
||||
```json
|
||||
{"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}}
|
||||
{"row":{"'hello'":"hello","multiply(42, number)":"42","range(5)":[0,1,2,3,4]}}
|
||||
{"row":{"'hello'":"hello","multiply(42, number)":"84","range(5)":[0,1,2,3,4]}}
|
||||
{"progress":{"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"3"}}
|
||||
```
|
||||
|
||||
## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes}
|
||||
## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes}
|
||||
|
||||
与`JSONCompactEachRow`/`JSONCompactStringEachRow`不同的是，其中列名和类型被写入前两行。
|
||||
|
||||
```json
|
||||
["'hello'", "multiply(42, number)", "range(5)"]
|
||||
["String", "UInt64", "Array(UInt8)"]
|
||||
["hello", "0", [0,1,2,3,4]]
|
||||
["hello", "42", [0,1,2,3,4]]
|
||||
["hello", "84", [0,1,2,3,4]]
|
||||
```
|
||||
|
||||
### Inserting Data {#inserting-data}
|
||||
|
||||
``` sql
|
||||
INSERT INTO UserActivity FORMAT JSONEachRow {"PageViews":5, "UserID":"4324182021466249494", "Duration":146,"Sign":-1} {"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1}
|
||||
```
|
||||
|
||||
ClickHouse允许:
|
||||
|
||||
- 对象中key-value的任何顺序。
|
||||
- 省略一些值。
|
||||
|
||||
ClickHouse忽略元素之间的空格和对象后面的逗号。您可以在一行中传递所有对象。你不需要用换行符把它们分开。
|
||||
|
||||
**省略值处理**
|
||||
|
||||
ClickHouse将省略的值替换为对应的[data types](../sql-reference/data-types/index.md)默认值。
|
||||
|
||||
如果指定了`DEFAULT expr`，则ClickHouse根据属性使用不同的替换规则，详看[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)设置。
|
||||
|
||||
参考下表：
|
||||
|
||||
``` sql
|
||||
CREATE TABLE IF NOT EXISTS example_table
|
||||
(
|
||||
x UInt32,
|
||||
a DEFAULT x * 2
|
||||
) ENGINE = Memory;
|
||||
```
|
||||
|
||||
- 如果`input_format_defaults_for_omitted_fields = 0`, 那么`x`和`a`的默认值等于`0`(作为`UInt32`数据类型的默认值)。
|
||||
- 如果`input_format_defaults_for_omitted_fields = 1`, 那么`x`的默认值为`0`，但`a`的默认值为`x * 2`。
|
||||
|
||||
!!! note "注意"
|
||||
当使用`insert_sample_with_metadata = 1`插入数据时，与使用`insert_sample_with_metadata = 0`插入数据相比，ClickHouse消耗更多的计算资源。
|
||||
|
||||
### Selecting Data {#selecting-data}
|
||||
|
||||
以`UserActivity`表为例:
|
||||
|
||||
``` text
|
||||
┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐
|
||||
│ 4324182021466249494 │ 5 │ 146 │ -1 │
|
||||
│ 4324182021466249494 │ 6 │ 185 │ 1 │
|
||||
└─────────────────────┴───────────┴──────────┴──────┘
|
||||
```
|
||||
|
||||
当查询`SELECT * FROM UserActivity FORMAT JSONEachRow`返回:
|
||||
|
||||
``` text
|
||||
{"UserID":"4324182021466249494","PageViews":5,"Duration":146,"Sign":-1}
|
||||
{"UserID":"4324182021466249494","PageViews":6,"Duration":185,"Sign":1}
|
||||
```
|
||||
|
||||
与[JSON](#json)格式不同，没有替换无效的UTF-8序列。值以与`JSON`相同的方式转义。
|
||||
|
||||
!!! note "提示"
|
||||
字符串中可以输出任意一组字节。如果您确信表中的数据可以被格式化为JSON而不会丢失任何信息，那么就使用`JSONEachRow`格式。
|
||||
|
||||
### Nested Structures {#jsoneachrow-nested}
|
||||
|
||||
如果您有一个包含[Nested](../sql-reference/data-types/nested-data-structures/nested.md)数据类型列的表，您可以插入具有相同结构的JSON数据。使用[input_format_import_nested_json](../operations/settings/settings.md#settings-input_format_import_nested_json)设置启用该特性。
|
||||
|
||||
例如，请参考下表:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE json_each_row_nested (n Nested (s String, i Int32) ) ENGINE = Memory
|
||||
```
|
||||
|
||||
正如你可以在找到 `Nested` 数据类型说明，ClickHouse将嵌套结构的每个组件视为单独的列, `n.s` 和 `n.i` 为了我们的桌子 所以你可以通过以下方式插入数据:
|
||||
正如您在`Nested`数据类型描述中看到的，ClickHouse将嵌套结构的每个组件作为一个单独的列(`n.s`和`n.i`是我们的表)。您可以通过以下方式插入数据:
|
||||
|
||||
``` sql
|
||||
INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i": [1, 23]}
|
||||
```
|
||||
|
||||
将数据作为分层JSON对象集插入 [input_format_import_nested_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json).
|
||||
将数据作为分层JSON对象集插入[input_format_import_nested_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json)。
|
||||
|
||||
``` json
|
||||
{
|
||||
@ -508,42 +762,59 @@ SELECT * FROM json_each_row_nested
|
||||
└───────────────┴────────┘
|
||||
```
|
||||
|
||||
## 本地人 {#native}
|
||||
## Native {#native}
|
||||
|
||||
最高性能的格式。 据通过二进制格式的块进行写入和读取。对于每个块，该块中的行数，列数，列名称和类型以及列的部分将被相继记录。 换句话说，这种格式是 «列式»的 - 它不会将列转换为行。 这是用于在服务器之间进行交互的本地界面中使用的格式，用于使用命令行客户端和 C++ 客户端。
|
||||
最高性能的格式。通过二进制格式的块进行写入和读取。对于每个块，该中的行数，列数，列名称和类型以及列的部分将被相继记录。 换句话说，这种格式是`columnar`的 - 它不会将列转换为行。这是用于在服务器之间进行交互的本地界面中使用的格式，用于使用命令行客户端和C++客户端。
|
||||
|
||||
您可以使用此格式快速生成只能由ClickHouse DBMS读取的格式。但自己处理这种格式是没有意义的。
|
||||
|
||||
## Null {#null}
|
||||
|
||||
没有输出。但是，查询已处理完毕，并且在使用命令行客户端时，数据将传输到客户端。这仅用于测试，包括生产力测试。
|
||||
没有输出。但是，查询已处理完毕，并且在使用命令行客户端时，数据将传输到客户端。这仅用于测试，包括性能测试。
|
||||
显然，这种格式只适用于输出，不适用于解析。
|
||||
|
||||
## 漂亮 {#pretty}
|
||||
## Pretty {#pretty}
|
||||
|
||||
将数据以表格形式输出，也可以使用ANSI转义字符在终端中设置颜色。
|
||||
它会绘制一个完整的表格，每行数据在终端中占用两行。
|
||||
每一个结果块都会以单独的表格输出。这是很有必要的，以便结果块不用缓冲结果输出（缓冲在可以预见结果集宽度的时候是很有必要的）。
|
||||
每个结果块作为一个单独的表输出。这是必要的，以便在输出块时不需要缓冲结果(为了预先计算所有值的可见宽度，缓冲是必要的)。
|
||||
|
||||
[NULL](../sql-reference/syntax.md)输出为`ᴺᵁᴸᴸ`。
|
||||
|
||||
示例(显示[PrettyCompact](#prettycompact)格式)
|
||||
|
||||
``` sql
|
||||
SELECT * FROM t_null
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─x─┬────y─┐
|
||||
│ 1 │ ᴺᵁᴸᴸ │
|
||||
└───┴──────┘
|
||||
```
|
||||
|
||||
为避免将太多数据传输到终端，只打印前10,000行。 如果行数大于或等于10,000，则会显示消息«Showed first 10 000»。
|
||||
行没有转义为Pretty\* 格式。示例显示了[PrettyCompact](#prettycompact)格式:
|
||||
|
||||
``` sql
|
||||
SELECT 'String with \'quotes\' and \t character' AS Escaping_test
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─Escaping_test────────────────────────┐
|
||||
│ String with 'quotes' and character │
|
||||
└──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
为避免将太多数据传输到终端，只打印前10,000行。 如果行数大于或等于10,000，则会显示消息`Showed first 10 000`。
|
||||
该格式仅适用于输出查询结果，但不适用于解析输入（将数据插入到表中）。
|
||||
|
||||
Pretty格式支持输出总值（当使用 WITH TOTALS 时）和极值（当 `extremes` 设置为1时）。 在这些情况下，总数值和极值在主数据之后以单独的表格形式输出。 示例（以 PrettyCompact 格式显示）：
|
||||
Pretty格式支持输出合计值(当使用WITH TOTALS时)和极值(当`extremes`设置为1时)。在这些情况下，合计值和极值将输出在主要数据之后，在单独的表中。示例(显示为[PrettyCompact](#prettycompact)格式):
|
||||
|
||||
``` sql
|
||||
SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact
|
||||
```
|
||||
|
||||
``` text
|
||||
┌──EventDate─┬───────c─┐
|
||||
│ 2014-03-17 │ 1406958 │
|
||||
│ 2014-03-18 │ 1383658 │
|
||||
@ -564,14 +835,16 @@ SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORD
|
||||
│ 2014-03-17 │ 1031592 │
|
||||
│ 2014-03-23 │ 1406958 │
|
||||
└────────────┴─────────┘
|
||||
```
|
||||
|
||||
## PrettyCompact {#prettycompact}
|
||||
|
||||
与 `Pretty` 格式不一样的是，`PrettyCompact` 去掉了行之间的表格分割线，这样使得结果更加紧凑。这种格式会在交互命令行客户端下默认使用。
|
||||
与[Pretty](#pretty)格式不一样的是`PrettyCompact`去掉了行之间的表格分割线，这样使得结果更加紧凑。
|
||||
这种格式会在交互命令行客户端下默认使用。
|
||||
|
||||
## PrettyCompactMonoBlock {#prettycompactmonoblock}
|
||||
|
||||
与 `PrettyCompact` 格式不一样的是，它支持 10,000 行数据缓冲，然后输出在一个表格中，不会按照块来区分
|
||||
与[PrettyCompact](#prettycompact)格式不一样的是，它支持10,000行数据缓冲，然后输出在一个表格中，不会按照块来区分。
|
||||
|
||||
## PrettyNoEscapes {#prettynoescapes}
|
||||
|
||||
@ -593,9 +866,17 @@ watch -n1 "clickhouse-client --query='SELECT event, value FROM system.events FOR
|
||||
|
||||
用法类似上述。
|
||||
|
||||
### PrettyCompactNoEscapes {#prettycompactnoescapes}
|
||||
|
||||
与前面的设置相同。
|
||||
|
||||
### PrettySpaceNoEscapes {#prettyspacenoescapes}
|
||||
|
||||
与前面的设置相同。
|
||||
|
||||
## PrettySpace {#prettyspace}
|
||||
|
||||
与 `PrettyCompact`(#prettycompact) 格式不一样的是，它使用空格来代替网格来显示数据。
|
||||
与[PrettyCompact](#prettycompact)格式不一样的是，它使用空格来代替网格来显示数据。
|
||||
|
||||
## RowBinary {#rowbinary}
|
||||
|
||||
|
||||
@ -1,51 +1,66 @@
|
||||
# 第三方开发的库 {#di-san-fang-kai-fa-de-ku}
|
||||
---
|
||||
toc_priority: 26
|
||||
toc_title: 客户端开发库
|
||||
---
|
||||
|
||||
!!! warning "放弃"
|
||||
Yandex不维护下面列出的库，也没有进行任何广泛的测试以确保其质量。
|
||||
# 第三方开发库 {#client-libraries-from-third-party-developers}
|
||||
|
||||
!!! warning "声明"
|
||||
Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试来确保它们的质量。
|
||||
|
||||
- Python
|
||||
- [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)
|
||||
- [ﾂ环板driverｮﾂ嘉ｯﾂ偲](https://github.com/mymarilyn/clickhouse-driver)
|
||||
- [ﾂ环板clientｮﾂ嘉ｯﾂ偲](https://github.com/yurial/clickhouse-client)
|
||||
- [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
|
||||
- [clickhouse-client](https://github.com/yurial/clickhouse-client)
|
||||
- [aiochclient](https://github.com/maximdanilchenko/aiochclient)
|
||||
- PHP
|
||||
- [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
|
||||
- [8bitov/clickhouse-php客户端](https://packagist.org/packages/8bitov/clickhouse-php-client)
|
||||
- [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://packagist.org/packages/bozerkins/clickhouse-client)
|
||||
- [ﾂ环板clientｮﾂ嘉ｯﾂ偲](https://packagist.org/packages/simpod/clickhouse-client)
|
||||
- [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
|
||||
- [bozerkins/clickhouse-client](https://packagist.org/packages/bozerkins/clickhouse-client)
|
||||
- [simpod/clickhouse-client](https://packagist.org/packages/simpod/clickhouse-client)
|
||||
- [seva-code/php-click-house-client](https://packagist.org/packages/seva-code/php-click-house-client)
|
||||
- [ﾂ环板clientｮﾂ嘉ｯﾂ偲](https://github.com/SeasX/SeasClick)
|
||||
- 走吧
|
||||
- [SeasClick C++ client](https://github.com/SeasX/SeasClick)
|
||||
- [one-ck](https://github.com/lizhichao/one-ck)
|
||||
- [glushkovds/phpclickhouse-laravel](https://packagist.org/packages/glushkovds/phpclickhouse-laravel)
|
||||
- Go
|
||||
- [clickhouse](https://github.com/kshvakov/clickhouse/)
|
||||
- [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/roistat/go-clickhouse)
|
||||
- [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/mailru/go-clickhouse)
|
||||
- [go-clickhouse](https://github.com/roistat/go-clickhouse)
|
||||
- [mailrugo-clickhouse](https://github.com/mailru/go-clickhouse)
|
||||
- [golang-clickhouse](https://github.com/leprosus/golang-clickhouse)
|
||||
- Swift
|
||||
- [ClickHouseNIO](https://github.com/patrick-zippenfenig/ClickHouseNIO)
|
||||
- [ClickHouseVapor ORM](https://github.com/patrick-zippenfenig/ClickHouseVapor)
|
||||
- NodeJs
|
||||
- [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人)](https://github.com/TimonKK/clickhouse)
|
||||
- [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/apla/node-clickhouse)
|
||||
- [clickhouse (NodeJs)](https://github.com/TimonKK/clickhouse)
|
||||
- [node-clickhouse](https://github.com/apla/node-clickhouse)
|
||||
- Perl
|
||||
- [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse)
|
||||
- [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse)
|
||||
- [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://metacpan.org/release/AnyEvent-ClickHouse)
|
||||
- [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse)
|
||||
- Ruby
|
||||
- [ﾂ暗ｪﾂ氾环催ﾂ団)](https://github.com/shlima/click_house)
|
||||
- [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/PNixx/clickhouse-activerecord)
|
||||
- [ClickHouse (Ruby)](https://github.com/shlima/click_house)
|
||||
- [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord)
|
||||
- R
|
||||
- [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r)
|
||||
- [RClickhouse](https://github.com/IMSMWU/RClickhouse)
|
||||
- [RClickHouse](https://github.com/IMSMWU/RClickHouse)
|
||||
- Java
|
||||
- [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java)
|
||||
- 斯卡拉
|
||||
- [掳胫client-禄脢鹿脷露胫鲁隆鹿-client酶](https://github.com/crobox/clickhouse-scala-client)
|
||||
- [clickhouse-client](https://github.com/Ecwid/clickhouse-client)
|
||||
- Scala
|
||||
- [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client)
|
||||
- Kotlin
|
||||
- [AORM](https://github.com/TanVD/AORM)
|
||||
- C#
|
||||
- [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient)
|
||||
- [克莱克豪斯Ado](https://github.com/killwort/ClickHouse-Net)
|
||||
- [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net)
|
||||
- [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client)
|
||||
- [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net)
|
||||
- [克莱克豪斯客户](https://github.com/DarkWanderer/ClickHouse.Client)
|
||||
- 仙丹
|
||||
- Elixir
|
||||
- [clickhousex](https://github.com/appodeal/clickhousex/)
|
||||
- 尼姆
|
||||
- [pillar](https://github.com/sofakingworld/pillar)
|
||||
- Nim
|
||||
- [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse)
|
||||
- Haskell
|
||||
- [hdbc-clickhouse](https://github.com/zaneli/hdbc-clickhouse)
|
||||
|
||||
[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/client_libraries/) <!--hide-->
|
||||
[来源文章](https://clickhouse.tech/docs/en/interfaces/third-party/client_libraries/) <!--hide-->
|
||||
|
||||
14
docs/zh/interfaces/third-party/index.md
vendored
14
docs/zh/interfaces/third-party/index.md
vendored
@ -1,8 +1,16 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_folder_title: "\u7B2C\u4E09\u65B9"
|
||||
toc_folder_title: 第三方工具
|
||||
toc_priority: 24
|
||||
---
|
||||
|
||||
# 第三方工具 {#third-party-interfaces}
|
||||
|
||||
这是第三方工具的链接集合，它们提供了一些ClickHouse的接口。它可以是可视化界面、命令行界面或API:
|
||||
|
||||
- [Client libraries](../../interfaces/third-party/client-libraries.md)
|
||||
- [Integrations](../../interfaces/third-party/integrations.md)
|
||||
- [GUI](../../interfaces/third-party/gui.md)
|
||||
- [Proxies](../../interfaces/third-party/proxy.md)
|
||||
|
||||
!!! note "注意"
|
||||
支持通用API的通用工具[ODBC](../../interfaces/odbc.md)或[JDBC](../../interfaces/jdbc.md)，通常也适用于ClickHouse，但这里没有列出，因为它们实在太多了。
|
||||
|
||||
86
docs/zh/interfaces/third-party/integrations.md
vendored
86
docs/zh/interfaces/third-party/integrations.md
vendored
@ -1,100 +1,108 @@
|
||||
# 第三方集成库 {#di-san-fang-ji-cheng-ku}
|
||||
---
|
||||
toc_priority: 27
|
||||
toc_title: 第三方集成库
|
||||
---
|
||||
|
||||
# 第三方集成库 {#integration-libraries-from-third-party-developers}
|
||||
|
||||
!!! warning "声明"
|
||||
Yandex不维护下面列出的库，也没有进行任何广泛的测试以确保其质量。
|
||||
Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试来确保它们的质量。
|
||||
|
||||
## 基建产品 {#ji-jian-chan-pin}
|
||||
## 基础设施 {#infrastructure-products}
|
||||
|
||||
- 关系数据库管理系统
|
||||
- 关系数据库
|
||||
- [MySQL](https://www.mysql.com)
|
||||
- [mysql2ch](https://github.com/long2ice/mysql2ch)
|
||||
- [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support)
|
||||
- [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader)
|
||||
- [horgh-复制器](https://github.com/larsnovikov/horgh-replicator)
|
||||
- [horgh-replicator](https://github.com/larsnovikov/horgh-replicator)
|
||||
- [PostgreSQL](https://www.postgresql.org)
|
||||
- [clickhousedb_fdw](https://github.com/Percona-Lab/clickhousedb_fdw)
|
||||
- [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (使用 [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
|
||||
- [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (uses [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
|
||||
- [pg2ch](https://github.com/mkabilov/pg2ch)
|
||||
- [clickhouse_fdw](https://github.com/adjust/clickhouse_fdw)
|
||||
- [MSSQL](https://en.wikipedia.org/wiki/Microsoft_SQL_Server)
|
||||
- [ClickHouseMightrator](https://github.com/zlzforever/ClickHouseMigrator)
|
||||
- [ClickHouseMigrator](https://github.com/zlzforever/ClickHouseMigrator)
|
||||
- 消息队列
|
||||
- [卡夫卡](https://kafka.apache.org)
|
||||
- [clickhouse_sinker](https://github.com/housepower/clickhouse_sinker) (使用 [去客户](https://github.com/ClickHouse/clickhouse-go/))
|
||||
- [Kafka](https://kafka.apache.org)
|
||||
- [clickhouse_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/ClickHouse/clickhouse-go/))
|
||||
- [stream-loader-clickhouse](https://github.com/adform/stream-loader)
|
||||
- 流处理
|
||||
- [Flink](https://flink.apache.org)
|
||||
- [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink)
|
||||
- 对象存储
|
||||
- [S3](https://en.wikipedia.org/wiki/Amazon_S3)
|
||||
- [ﾂ环板backupｮﾂ嘉ｯﾂ偲](https://github.com/AlexAkulov/clickhouse-backup)
|
||||
- [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup)
|
||||
- 容器编排
|
||||
- [Kubernetes](https://kubernetes.io)
|
||||
- [clickhouse-操](https://github.com/Altinity/clickhouse-operator)
|
||||
- [clickhouse-operator](https://github.com/Altinity/clickhouse-operator)
|
||||
- 配置管理
|
||||
- [木偶](https://puppet.com)
|
||||
- [ﾂ环板/ｮﾂ嘉ｯﾂ偲](https://forge.puppet.com/innogames/clickhouse)
|
||||
- [puppet](https://puppet.com)
|
||||
- [innogames/clickhouse](https://forge.puppet.com/innogames/clickhouse)
|
||||
- [mfedotov/clickhouse](https://forge.puppet.com/mfedotov/clickhouse)
|
||||
- 监控
|
||||
- [石墨](https://graphiteapp.org)
|
||||
- Monitoring
|
||||
- [Graphite](https://graphiteapp.org)
|
||||
- [graphouse](https://github.com/yandex/graphouse)
|
||||
- [ﾂ暗ｪﾂ氾环催ﾂ団](https://github.com/lomik/carbon-clickhouse) +
|
||||
- [ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/lomik/graphite-clickhouse)
|
||||
- [石墨-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) -优化静态分区 [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) 如果从规则 [汇总配置](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) 可以应用
|
||||
- [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) +
|
||||
- [graphite-clickhouse](https://github.com/lomik/graphite-clickhouse)
|
||||
- [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer) - optimizes staled partitions in [\*GraphiteMergeTree](../../engines/table-engines/mergetree-family/graphitemergetree.md#graphitemergetree) if rules from [rollup configuration](../../engines/table-engines/mergetree-family/graphitemergetree.md#rollup-configuration) could be applied
|
||||
- [Grafana](https://grafana.com/)
|
||||
- [clickhouse-grafana](https://github.com/Vertamedia/clickhouse-grafana)
|
||||
- [普罗米修斯号](https://prometheus.io/)
|
||||
- [Prometheus](https://prometheus.io/)
|
||||
- [clickhouse_exporter](https://github.com/f1yegor/clickhouse_exporter)
|
||||
- [PromHouse](https://github.com/Percona-Lab/PromHouse)
|
||||
- [clickhouse_exporter](https://github.com/hot-wifi/clickhouse_exporter) （用途 [去客户](https://github.com/kshvakov/clickhouse/))
|
||||
- [clickhouse_exporter](https://github.com/hot-wifi/clickhouse_exporter) (uses [Go client](https://github.com/kshvakov/clickhouse/))
|
||||
- [Nagios](https://www.nagios.org/)
|
||||
- [check_clickhouse](https://github.com/exogroup/check_clickhouse/)
|
||||
- [check_clickhouse.py](https://github.com/innogames/igmonplugins/blob/master/src/check_clickhouse.py)
|
||||
- [Zabbix](https://www.zabbix.com)
|
||||
- [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/Altinity/clickhouse-zabbix-template)
|
||||
- [clickhouse-zabbix-template](https://github.com/Altinity/clickhouse-zabbix-template)
|
||||
- [Sematext](https://sematext.com/)
|
||||
- [clickhouse积分](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse)
|
||||
- 记录
|
||||
- [clickhouse integration](https://github.com/sematext/sematext-agent-integrations/tree/master/clickhouse)
|
||||
- Logging
|
||||
- [rsyslog](https://www.rsyslog.com/)
|
||||
- [鹿茫house omhousee酶](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html)
|
||||
- [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html)
|
||||
- [fluentd](https://www.fluentd.org)
|
||||
- [loghouse](https://github.com/flant/loghouse) (对于 [Kubernetes](https://kubernetes.io))
|
||||
- [Sematext](https://www.sematext.com/logagent)
|
||||
- [logagent输出-插件-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/)
|
||||
- 地理
|
||||
- [loghouse](https://github.com/flant/loghouse) (for [Kubernetes](https://kubernetes.io))
|
||||
- [logagent](https://www.sematext.com/logagent)
|
||||
- [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/)
|
||||
- Geo
|
||||
- [MaxMind](https://dev.maxmind.com/geoip/)
|
||||
- [ﾂ环板-ｮﾂ嘉ｯﾂ偲青clickｼﾂ氾ｶﾂ鉄ﾂ工ﾂ渉](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
|
||||
- [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
|
||||
|
||||
## 编程语言生态系统 {#bian-cheng-yu-yan-sheng-tai-xi-tong}
|
||||
## 编程语言 {#programming-language-ecosystems}
|
||||
|
||||
- Python
|
||||
- [SQLAlchemy](https://www.sqlalchemy.org)
|
||||
- [ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人](https://github.com/cloudflare/sqlalchemy-clickhouse) (使用 [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
|
||||
- [熊猫](https://pandas.pydata.org)
|
||||
- [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) (uses [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
|
||||
- [pandas](https://pandas.pydata.org)
|
||||
- [pandahouse](https://github.com/kszucs/pandahouse)
|
||||
- PHP
|
||||
- [Doctrine](https://www.doctrine-project.org/)
|
||||
- [dbal-clickhouse](https://packagist.org/packages/friendsofdoctrine/dbal-clickhouse)
|
||||
- R
|
||||
- [dplyr](https://db.rstudio.com/dplyr/)
|
||||
- [RClickhouse](https://github.com/IMSMWU/RClickhouse) (使用 [ﾂ暗ｪﾂ氾环催ﾂ団](https://github.com/artpaul/clickhouse-cpp))
|
||||
- [RClickHouse](https://github.com/IMSMWU/RClickHouse) (uses [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp))
|
||||
- Java
|
||||
- [Hadoop](http://hadoop.apache.org)
|
||||
- [clickhouse-hdfs-装载机](https://github.com/jaykelin/clickhouse-hdfs-loader) (使用 [JDBC](../../sql-reference/table-functions/jdbc.md))
|
||||
- 斯卡拉
|
||||
- [clickhouse-hdfs-loader](https://github.com/jaykelin/clickhouse-hdfs-loader) (uses [JDBC](../../sql-reference/table-functions/jdbc.md))
|
||||
- Scala
|
||||
- [Akka](https://akka.io)
|
||||
- [掳胫client-禄脢鹿脷露胫鲁隆鹿-client酶](https://github.com/crobox/clickhouse-scala-client)
|
||||
- [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client)
|
||||
- C#
|
||||
- [ADO.NET](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/ado-net-overview)
|
||||
- [克莱克豪斯Ado](https://github.com/killwort/ClickHouse-Net)
|
||||
- [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net)
|
||||
- [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client)
|
||||
- [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net)
|
||||
- [ClickHouse.Net.Migrations](https://github.com/ilyabreev/ClickHouse.Net.Migrations)
|
||||
- 仙丹
|
||||
- Elixir
|
||||
- [Ecto](https://github.com/elixir-ecto/ecto)
|
||||
- [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto)
|
||||
- Ruby
|
||||
- [Ruby on Rails](https://rubyonrails.org/)
|
||||
- [activecube](https://github.com/bitquery/activecube)
|
||||
- [ActiveRecord](https://github.com/PNixx/clickhouse-activerecord)
|
||||
- [GraphQL](https://github.com/graphql)
|
||||
- [activecube-graphql](https://github.com/bitquery/activecube-graphql)
|
||||
|
||||
[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/integrations/) <!--hide-->
|
||||
[源文章](https://clickhouse.tech/docs/en/interfaces/third-party/integrations/) <!--hide-->
|
||||
|
||||
49
docs/zh/interfaces/third-party/proxy.md
vendored
49
docs/zh/interfaces/third-party/proxy.md
vendored
@ -1,37 +1,44 @@
|
||||
# 来自第三方开发人员的代理服务器 {#lai-zi-di-san-fang-kai-fa-ren-yuan-de-dai-li-fu-wu-qi}
|
||||
---
|
||||
toc_priority: 29
|
||||
toc_title: 第三方代理
|
||||
---
|
||||
|
||||
[chproxy](https://github.com/Vertamedia/chproxy) 是ClickHouse数据库的http代理和负载均衡器。
|
||||
# 第三方代理 {#proxy-servers-from-third-party-developers}
|
||||
|
||||
特征
|
||||
## chproxy {#chproxy}
|
||||
|
||||
*每用户路由和响应缓存。
|
||||
*灵活的限制。
|
||||
\*自动SSL证书续订。
|
||||
[chproxy](https://github.com/Vertamedia/chproxy), 是一个用于ClickHouse数据库的HTTP代理和负载均衡器。
|
||||
|
||||
在Go中实现。
|
||||
特性:
|
||||
|
||||
- 用户路由和响应缓存。
|
||||
- 灵活的限制。
|
||||
- 自动SSL证书续订。
|
||||
|
||||
使用go语言实现。
|
||||
|
||||
## KittenHouse {#kittenhouse}
|
||||
|
||||
[KittenHouse](https://github.com/VKCOM/kittenhouse) 设计为ClickHouse和应用程序服务器之间的本地代理，以防在应用程序端缓冲INSERT数据是不可能或不方便的。
|
||||
[KittenHouse](https://github.com/VKCOM/kittenhouse)被设计为ClickHouse和应用服务器之间的本地代理，以防不可能或不方便在应用程序端缓冲插入数据。
|
||||
|
||||
特征：
|
||||
特性:
|
||||
|
||||
*内存和磁盘数据缓冲。
|
||||
*每表路由。
|
||||
\*负载平衡和健康检查。
|
||||
- 内存和磁盘上的数据缓冲。
|
||||
- 表路由。
|
||||
- 负载平衡和运行状况检查。
|
||||
|
||||
在Go中实现。
|
||||
使用go语言实现。
|
||||
|
||||
## ﾂ环板-ｮﾂ嘉ｯﾂ偲 {#clickhouse-bulk}
|
||||
## ClickHouse-Bulk {#clickhouse-bulk}
|
||||
|
||||
[ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/nikepan/clickhouse-bulk) 是一个简单的ClickHouse插入收集器。
|
||||
[ClickHouse-Bulk](https://github.com/nikepan/clickhouse-bulk)是一个简单的ClickHouse收集器。
|
||||
|
||||
特征：
|
||||
特性:
|
||||
|
||||
*分组请求并按阈值或间隔发送。
|
||||
*多个远程服务器。
|
||||
\*基本身份验证。
|
||||
- 按阈值或间隔对请求进行分组并发送。
|
||||
- 多个远程服务器。
|
||||
- 基本身份验证。
|
||||
|
||||
在Go中实现。
|
||||
使用go语言实现。
|
||||
|
||||
[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/proxy/) <!--hide-->
|
||||
[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/proxy/) <!--hide-->
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_folder_title: "\u65B0\u589E\u5185\u5BB9"
|
||||
toc_priority: 72
|
||||
toc_folder_title: ClickHouse事迹
|
||||
toc_priority: 82
|
||||
---
|
||||
|
||||
# ClickHouse变更及改动? {#whats-new-in-clickhouse}
|
||||
|
||||
对于已经发布的版本，有一个[roadmap](../whats-new/roadmap.md)和一个详细的[changelog](../whats-new/changelog/index.md)。
|
||||
|
||||
@ -1,17 +1,10 @@
|
||||
---
|
||||
toc_priority: 74
|
||||
toc_title: 路线图
|
||||
toc_title: Roadmap
|
||||
---
|
||||
|
||||
# 路线图 {#roadmap}
|
||||
# Roadmap {#roadmap}
|
||||
|
||||
## Q2 2020 {#q2-2020}
|
||||
|
||||
- 和外部认证服务集成
|
||||
|
||||
## Q3 2020 {#q3-2020}
|
||||
|
||||
- 资源池，为用户提供更精准的集群资源分配
|
||||
|
||||
{## [原始文档](https://clickhouse.tech/docs/en/roadmap/) ##}
|
||||
`2021年Roadmap`已公布供公开讨论查看[这里](https://github.com/ClickHouse/ClickHouse/issues/17623).
|
||||
|
||||
{## [源文章](https://clickhouse.tech/docs/en/roadmap/) ##}
|
||||
|
||||
@ -1,41 +1,74 @@
|
||||
## 修复于 ClickHouse Release 18.12.13, 2018-09-10 {#xiu-fu-yu-clickhouse-release-18-12-13-2018-09-10}
|
||||
---
|
||||
toc_priority: 76
|
||||
toc_title: 安全更新日志
|
||||
---
|
||||
|
||||
## 修复于ClickHouse Release 19.14.3.3, 2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10}
|
||||
|
||||
### CVE-2019-15024 {#cve-2019-15024}
|
||||
|
||||
对ZooKeeper具有写访问权限并且可以运行ClickHouse所在网络上可用的自定义服务器的攻击者可以创建一个自定义的恶意服务器，该服务器将充当ClickHouse副本并在ZooKeeper中注册。当另一个副本将从恶意副本获取数据部分时，它可以强制clickhouse服务器写入文件系统上的任意路径。
|
||||
|
||||
作者：Yandex信息安全团队Eldar Zaitov
|
||||
|
||||
### CVE-2019-16535 {#cve-2019-16535}
|
||||
|
||||
解压算法中的OOB-read、OOB-write和整数下溢可以通过本机协议实现RCE或DoS。
|
||||
|
||||
作者: Yandex信息安全团队Eldar Zaitov
|
||||
|
||||
### CVE-2019-16536 {#cve-2019-16536}
|
||||
|
||||
恶意的经过身份验证的客户端可能会触发导致DoS的堆栈溢出。
|
||||
|
||||
作者: Yandex信息安全团队Eldar Zaitov
|
||||
|
||||
## 修复于ClickHouse Release 19.13.6.1, 2019-09-20 {#fixed-in-clickhouse-release-19-13-6-1-2019-09-20}
|
||||
|
||||
### CVE-2019-18657 {#cve-2019-18657}
|
||||
|
||||
表函数`url`存在允许攻击者在请求中插入任意HTTP标头的漏洞。
|
||||
|
||||
作者: [Nikita Tikhomirov](https://github.com/NSTikhomirov)
|
||||
|
||||
## 修复于ClickHouse Release 18.12.13, 2018-09-10 {#fixed-in-clickhouse-release-18-12-13-2018-09-10}
|
||||
|
||||
### CVE-2018-14672 {#cve-2018-14672}
|
||||
|
||||
加载CatBoost模型的功能，允许遍历路径并通过错误消息读取任意文件。
|
||||
加载CatBoost模型的函数允许路径遍历和通过错误消息读取任意文件。
|
||||
|
||||
来源: Yandex信息安全团队的Andrey Krasichkov
|
||||
作者：Yandex信息安全团队Andrey Krasichkov
|
||||
|
||||
## 修复于 ClickHouse Release 18.10.3, 2018-08-13 {#xiu-fu-yu-clickhouse-release-18-10-3-2018-08-13}
|
||||
## 修复于Release 18.10.3, 2018-08-13 {#fixed-in-clickhouse-release-18-10-3-2018-08-13}
|
||||
|
||||
### CVE-2018-14671 {#cve-2018-14671}
|
||||
|
||||
unixODBC允许从文件系统加载任意共享对象，从而导致«远程执行代码»漏洞。
|
||||
unixODBC允许从文件系统加载任意共享对象，从而导致远程代码执行漏洞。
|
||||
|
||||
来源：Yandex信息安全团队的Andrey Krasichkov和Evgeny Sidorov
|
||||
作者：Yandex信息安全团队Andrey Krasichkov和Evgeny Sidorov
|
||||
|
||||
## 修复于 ClickHouse Release 1.1.54388, 2018-06-28 {#xiu-fu-yu-clickhouse-release-1-1-54388-2018-06-28}
|
||||
## 修复于ClickHouse Release 1.1.54388, 2018-06-28 {#fixed-in-clickhouse-release-1-1-54388-2018-06-28}
|
||||
|
||||
### CVE-2018-14668 {#cve-2018-14668}
|
||||
|
||||
远程表函数功能允许在 «user», «password» 及 «default_database» 字段中使用任意符号，从而导致跨协议请求伪造攻击。
|
||||
`remote`表函数允许在`user`，`password`和`default_database`字段中使用任意符号，从而导致跨协议请求伪造攻击。
|
||||
|
||||
来源：Yandex信息安全团队的Andrey Krasichkov
|
||||
者：Yandex信息安全团队Andrey Krasichkov
|
||||
|
||||
## 修复于 ClickHouse Release 1.1.54390, 2018-07-06 {#xiu-fu-yu-clickhouse-release-1-1-54390-2018-07-06}
|
||||
## 修复于ClickHouse Release 1.1.54390, 2018-07-06 {#fixed-in-clickhouse-release-1-1-54390-2018-07-06}
|
||||
|
||||
### CVE-2018-14669 {#cve-2018-14669}
|
||||
|
||||
ClickHouse MySQL客户端启用了 «LOAD DATA LOCAL INFILE» 功能，该功能允许恶意MySQL数据库从连接的ClickHouse服务器读取任意文件。
|
||||
ClickHouse MySQL客户端启用了`LOAD DATA LOCAL INFILE`功能，允许恶意MySQL数据库从连接的ClickHouse服务器读取任意文件。
|
||||
|
||||
来源：Yandex信息安全团队的Andrey Krasichkov和Evgeny Sidorov
|
||||
作者：Yandex信息安全团队Andrey Krasichkov和Evgeny Sidorov
|
||||
|
||||
## 修复于 ClickHouse Release 1.1.54131, 2017-01-10 {#xiu-fu-yu-clickhouse-release-1-1-54131-2017-01-10}
|
||||
## 修复于ClickHouse Release 1.1.54131, 2017-01-10 {#fixed-in-clickhouse-release-1-1-54131-2017-01-10}
|
||||
|
||||
### CVE-2018-14670 {#cve-2018-14670}
|
||||
|
||||
deb软件包中的错误配置可能导致使用未经授权的数据库。
|
||||
deb包中的错误配置可能导致未经授权使用数据库。
|
||||
|
||||
来源：英国国家网络安全中心（NCSC）
|
||||
作者：英国国家网络安全中心（NCSC）
|
||||
|
||||
[来源文章](https://clickhouse.tech/docs/en/security_changelog/) <!--hide-->
|
||||
{## [Original article](https://clickhouse.tech/docs/en/security_changelog/) ##}
|
||||
|
||||
@ -405,8 +405,8 @@ void QueryFuzzer::fuzz(ASTPtr & ast)
|
||||
|
||||
if (fn->is_window_function)
|
||||
{
|
||||
fuzzColumnLikeExpressionList(fn->window_partition_by);
|
||||
fuzzOrderByList(fn->window_order_by);
|
||||
fuzzColumnLikeExpressionList(fn->window_partition_by.get());
|
||||
fuzzOrderByList(fn->window_order_by.get());
|
||||
}
|
||||
|
||||
fuzz(fn->children);
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeFixedString.h>
|
||||
#include <DataTypes/DataTypeUUID.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
@ -76,6 +77,8 @@ void ExternalResultDescription::init(const Block & sample_block_)
|
||||
types.emplace_back(ValueType::vtDecimal128, is_nullable);
|
||||
else if (typeid_cast<const DataTypeDecimal<Decimal256> *>(type))
|
||||
types.emplace_back(ValueType::vtDecimal256, is_nullable);
|
||||
else if (typeid_cast<const DataTypeFixedString *>(type))
|
||||
types.emplace_back(ValueType::vtFixedString, is_nullable);
|
||||
else
|
||||
throw Exception{"Unsupported type " + type->getName(), ErrorCodes::UNKNOWN_TYPE};
|
||||
}
|
||||
|
||||
@ -30,7 +30,8 @@ struct ExternalResultDescription
|
||||
vtDecimal32,
|
||||
vtDecimal64,
|
||||
vtDecimal128,
|
||||
vtDecimal256
|
||||
vtDecimal256,
|
||||
vtFixedString
|
||||
};
|
||||
|
||||
Block sample_block;
|
||||
|
||||
@ -518,7 +518,7 @@ void IPAddressDictionary::loadData()
|
||||
{
|
||||
/// We format key attribute values here instead of filling with data from key_column
|
||||
/// because string representation can be normalized if bits beyond mask are set.
|
||||
/// Also all IPv4 will be displayed as mapped IPv6 if threre are any IPv6.
|
||||
/// Also all IPv4 will be displayed as mapped IPv6 if there are any IPv6.
|
||||
/// It's consistent with representation in table created with `ENGINE = Dictionary` from this dictionary.
|
||||
char str_buffer[48];
|
||||
if (has_ipv6)
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
# include <Columns/ColumnString.h>
|
||||
# include <Columns/ColumnsNumber.h>
|
||||
# include <Columns/ColumnDecimal.h>
|
||||
# include <Columns/ColumnFixedString.h>
|
||||
# include <DataTypes/IDataType.h>
|
||||
# include <DataTypes/DataTypeNullable.h>
|
||||
# include <IO/ReadHelpers.h>
|
||||
@ -110,6 +111,9 @@ namespace
|
||||
data_type.deserializeAsWholeText(column, buffer, FormatSettings{});
|
||||
break;
|
||||
}
|
||||
case ValueType::vtFixedString:
|
||||
assert_cast<ColumnFixedString &>(column).insertData(value.data(), value.size());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -738,15 +738,26 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
|
||||
if (node.is_window_function)
|
||||
{
|
||||
// Also add columns from PARTITION BY and ORDER BY of window functions.
|
||||
// Requiring a constant reference to a shared pointer to non-const AST
|
||||
// doesn't really look sane, but the visitor does indeed require it.
|
||||
if (node.window_partition_by)
|
||||
{
|
||||
visit(node.window_partition_by->clone(), data);
|
||||
visit(node.window_partition_by, data);
|
||||
}
|
||||
if (node.window_order_by)
|
||||
{
|
||||
visit(node.window_order_by->clone(), data);
|
||||
visit(node.window_order_by, data);
|
||||
}
|
||||
|
||||
// Also manually add columns for arguments of the window function itself.
|
||||
// ActionVisitor is written in such a way that this method must itself
|
||||
// descend into all needed function children. Window functions can't have
|
||||
// any special functions as argument, so the code below that handles
|
||||
// special arguments is not needed. This is analogous to the
|
||||
// appendWindowFunctionsArguments() in SelectQueryExpressionAnalyzer and
|
||||
// partially duplicates its code. Probably we can remove most of the
|
||||
// logic from that function, but I don't yet have it all figured out...
|
||||
for (const auto & arg : node.arguments->children)
|
||||
{
|
||||
visit(arg, data);
|
||||
}
|
||||
|
||||
// Don't need to do anything more for window functions here -- the
|
||||
|
||||
@ -970,7 +970,9 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments(
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
|
||||
|
||||
// 1) Add actions for window functions and their arguments;
|
||||
// 2) Mark the columns that are really required.
|
||||
// 2) Mark the columns that are really required. We have to mark them as
|
||||
// required because we finish the expression chain before processing the
|
||||
// window functions.
|
||||
for (const auto & [_, w] : window_descriptions)
|
||||
{
|
||||
for (const auto & f : w.window_functions)
|
||||
@ -981,41 +983,14 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments(
|
||||
getRootActionsNoMakeSet(f.function_node->clone(),
|
||||
true /* no_subqueries */, step.actions());
|
||||
|
||||
// 1.2) result of window function: an empty INPUT.
|
||||
// It is an aggregate function, so it won't be added by getRootActions.
|
||||
// This is something of a hack. Other options:
|
||||
// a] do it like aggregate function -- break the chain of actions
|
||||
// and manually add window functions to the starting list of
|
||||
// input columns. Logically this is similar to what we're doing
|
||||
// now, but would require to split the window function processing
|
||||
// into a full-fledged step after plain functions. This would be
|
||||
// somewhat cumbersome. With INPUT hack we can avoid a separate
|
||||
// step and pretend that window functions are almost "normal"
|
||||
// select functions. The limitation of both these ways is that
|
||||
// we can't reference window functions in other SELECT
|
||||
// expressions.
|
||||
// b] add a WINDOW action type, then sort, then split the chain on
|
||||
// each WINDOW action and insert the Window pipeline between the
|
||||
// Expression pipelines. This is a "proper" way that would allow
|
||||
// us to depend on window functions in other functions. But it's
|
||||
// complicated so I avoid doing it for now.
|
||||
ColumnWithTypeAndName col;
|
||||
col.type = f.aggregate_function->getReturnType();
|
||||
col.column = col.type->createColumn();
|
||||
col.name = f.column_name;
|
||||
|
||||
step.actions()->addInput(col);
|
||||
|
||||
// 2.1) function arguments;
|
||||
for (const auto & a : f.function_node->arguments->children)
|
||||
{
|
||||
// 2.1) function arguments;
|
||||
step.required_output.push_back(a->getColumnName());
|
||||
}
|
||||
// 2.2) function result;
|
||||
step.required_output.push_back(f.column_name);
|
||||
}
|
||||
|
||||
// 2.3) PARTITION BY and ORDER BY columns.
|
||||
// 2.1) PARTITION BY and ORDER BY columns.
|
||||
for (const auto & c : w.full_sort_description)
|
||||
{
|
||||
step.required_output.push_back(c.column_name);
|
||||
@ -1048,6 +1023,15 @@ void SelectQueryExpressionAnalyzer::appendSelect(ExpressionActionsChain & chain,
|
||||
|
||||
for (const auto & child : select_query->select()->children)
|
||||
{
|
||||
if (const auto * function = typeid_cast<const ASTFunction *>(child.get());
|
||||
function
|
||||
&& function->is_window_function)
|
||||
{
|
||||
// Skip window function columns here -- they are calculated after
|
||||
// other SELECT expressions by a special step.
|
||||
continue;
|
||||
}
|
||||
|
||||
step.required_output.push_back(child->getColumnName());
|
||||
}
|
||||
}
|
||||
@ -1421,11 +1405,54 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
/// If there is aggregation, we execute expressions in SELECT and ORDER BY on the initiating server, otherwise on the source servers.
|
||||
query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage));
|
||||
|
||||
// Window functions are processed in a separate expression chain after
|
||||
// the main SELECT, similar to what we do for aggregate functions.
|
||||
if (has_window)
|
||||
{
|
||||
query_analyzer.appendWindowFunctionsArguments(chain, only_types || !first_stage);
|
||||
|
||||
// Build a list of output columns of the window step.
|
||||
// 1) We need the columns that are the output of ExpressionActions.
|
||||
for (const auto & x : chain.getLastActions()->getNamesAndTypesList())
|
||||
{
|
||||
query_analyzer.columns_after_window.push_back(x);
|
||||
}
|
||||
// 2) We also have to manually add the output of the window function
|
||||
// to the list of the output columns of the window step, because the
|
||||
// window functions are not in the ExpressionActions.
|
||||
for (const auto & [_, w] : query_analyzer.window_descriptions)
|
||||
{
|
||||
for (const auto & f : w.window_functions)
|
||||
{
|
||||
query_analyzer.columns_after_window.push_back(
|
||||
{f.column_name, f.aggregate_function->getReturnType()});
|
||||
}
|
||||
}
|
||||
|
||||
before_window = chain.getLastActions();
|
||||
finalize_chain(chain);
|
||||
|
||||
auto & step = chain.lastStep(query_analyzer.columns_after_window);
|
||||
|
||||
// The output of this expression chain is the result of
|
||||
// SELECT (before "final projection" i.e. renaming the columns), so
|
||||
// we have to mark the expressions that are required in the output,
|
||||
// again. We did it for the previous expression chain ("select w/o
|
||||
// window functions") earlier, in appendSelect(). But that chain also
|
||||
// produced the expressions required to calculate window functions.
|
||||
// They are not needed in the final SELECT result. Knowing the correct
|
||||
// list of columns is important when we apply SELECT DISTINCT later.
|
||||
const auto * select_query = query_analyzer.getSelectQuery();
|
||||
for (const auto & child : select_query->select()->children)
|
||||
{
|
||||
step.required_output.push_back(child->getColumnName());
|
||||
}
|
||||
}
|
||||
|
||||
selected_columns = chain.getLastStep().required_output;
|
||||
|
||||
has_order_by = query.orderBy() != nullptr;
|
||||
before_order_and_select = query_analyzer.appendOrderBy(
|
||||
before_order_by = query_analyzer.appendOrderBy(
|
||||
chain,
|
||||
only_types || (need_aggregate ? !second_stage : !first_stage),
|
||||
optimize_read_in_order,
|
||||
@ -1572,9 +1599,9 @@ std::string ExpressionAnalysisResult::dump() const
|
||||
ss << "before_window " << before_window->dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (before_order_and_select)
|
||||
if (before_order_by)
|
||||
{
|
||||
ss << "before_order_and_select " << before_order_and_select->dumpDAG() << "\n";
|
||||
ss << "before_order_by " << before_order_by->dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (before_limit_by)
|
||||
@ -1587,6 +1614,20 @@ std::string ExpressionAnalysisResult::dump() const
|
||||
ss << "final_projection " << final_projection->dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (!selected_columns.empty())
|
||||
{
|
||||
ss << "selected_columns ";
|
||||
for (size_t i = 0; i < selected_columns.size(); i++)
|
||||
{
|
||||
if (i > 0)
|
||||
{
|
||||
ss << ", ";
|
||||
}
|
||||
ss << backQuote(selected_columns[i]);
|
||||
}
|
||||
ss << "\n";
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
|
||||
@ -55,6 +55,8 @@ struct ExpressionAnalyzerData
|
||||
NamesAndTypesList columns_after_join;
|
||||
/// Columns after ARRAY JOIN, JOIN, and/or aggregation.
|
||||
NamesAndTypesList aggregated_columns;
|
||||
/// Columns after window functions.
|
||||
NamesAndTypesList columns_after_window;
|
||||
|
||||
bool has_aggregation = false;
|
||||
NamesAndTypesList aggregation_keys;
|
||||
@ -202,11 +204,12 @@ struct ExpressionAnalysisResult
|
||||
ActionsDAGPtr before_aggregation;
|
||||
ActionsDAGPtr before_having;
|
||||
ActionsDAGPtr before_window;
|
||||
ActionsDAGPtr before_order_and_select;
|
||||
ActionsDAGPtr before_order_by;
|
||||
ActionsDAGPtr before_limit_by;
|
||||
ActionsDAGPtr final_projection;
|
||||
|
||||
/// Columns from the SELECT list, before renaming them to aliases.
|
||||
/// Columns from the SELECT list, before renaming them to aliases. Used to
|
||||
/// perform SELECT DISTINCT.
|
||||
Names selected_columns;
|
||||
|
||||
/// Columns will be removed after prewhere actions execution.
|
||||
|
||||
@ -22,15 +22,22 @@ void ExpressionInfoMatcher::visit(const ASTFunction & ast_function, const ASTPtr
|
||||
{
|
||||
data.is_array_join = true;
|
||||
}
|
||||
// "is_aggregate_function" doesn't mean much by itself. Apparently here it is
|
||||
// used to move filters from HAVING to WHERE, and probably for this purpose
|
||||
// an aggregate function calculated as a window function is not relevant.
|
||||
// "is_aggregate_function" is used to determine whether we can move a filter
|
||||
// (1) from HAVING to WHERE or (2) from WHERE of a parent query to HAVING of
|
||||
// a subquery.
|
||||
// For aggregate functions we can't do (1) but can do (2).
|
||||
// For window functions both don't make sense -- they are not allowed in
|
||||
// WHERE or HAVING.
|
||||
else if (!ast_function.is_window_function
|
||||
&& AggregateFunctionFactory::instance().isAggregateFunctionName(
|
||||
ast_function.name))
|
||||
{
|
||||
data.is_aggregate_function = true;
|
||||
}
|
||||
else if (ast_function.is_window_function)
|
||||
{
|
||||
data.is_window_function = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto & function = FunctionFactory::instance().tryGet(ast_function.name, data.context);
|
||||
@ -75,16 +82,27 @@ bool ExpressionInfoMatcher::needChildVisit(const ASTPtr & node, const ASTPtr &)
|
||||
return !node->as<ASTSubquery>();
|
||||
}
|
||||
|
||||
bool hasStatefulFunction(const ASTPtr & node, const Context & context)
|
||||
bool hasNonRewritableFunction(const ASTPtr & node, const Context & context)
|
||||
{
|
||||
for (const auto & select_expression : node->children)
|
||||
{
|
||||
ExpressionInfoVisitor::Data expression_info{.context = context, .tables = {}};
|
||||
ExpressionInfoVisitor(expression_info).visit(select_expression);
|
||||
|
||||
if (expression_info.is_stateful_function)
|
||||
if (expression_info.is_stateful_function
|
||||
|| expression_info.is_window_function)
|
||||
{
|
||||
// If an outer query has a WHERE on window function, we can't move
|
||||
// it into the subquery, because window functions are not allowed in
|
||||
// WHERE and HAVING. Example:
|
||||
// select * from (
|
||||
// select number,
|
||||
// count(*) over (partition by intDiv(number, 3)) c
|
||||
// from numbers(3)
|
||||
// ) where c > 1;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -21,6 +21,7 @@ struct ExpressionInfoMatcher
|
||||
bool is_array_join = false;
|
||||
bool is_stateful_function = false;
|
||||
bool is_aggregate_function = false;
|
||||
bool is_window_function = false;
|
||||
bool is_deterministic_function = true;
|
||||
std::unordered_set<size_t> unique_reference_tables_pos = {};
|
||||
};
|
||||
@ -36,6 +37,6 @@ struct ExpressionInfoMatcher
|
||||
|
||||
using ExpressionInfoVisitor = ConstInDepthNodeVisitor<ExpressionInfoMatcher, true>;
|
||||
|
||||
bool hasStatefulFunction(const ASTPtr & node, const Context & context);
|
||||
bool hasNonRewritableFunction(const ASTPtr & node, const Context & context);
|
||||
|
||||
}
|
||||
|
||||
@ -33,11 +33,14 @@ public:
|
||||
return false;
|
||||
if (auto * func = node->as<ASTFunction>())
|
||||
{
|
||||
if (isAggregateFunction(*func)
|
||||
|| func->is_window_function)
|
||||
if (isAggregateFunction(*func))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Window functions can contain aggregation results as arguments
|
||||
// to the window functions, or columns of PARTITION BY or ORDER BY
|
||||
// of the window.
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -538,7 +538,10 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
|
||||
if (options.to_stage == QueryProcessingStage::Enum::WithMergeableState)
|
||||
{
|
||||
if (!analysis_result.need_aggregate)
|
||||
return analysis_result.before_order_and_select->getResultColumns();
|
||||
{
|
||||
// What's the difference with selected_columns?
|
||||
return analysis_result.before_order_by->getResultColumns();
|
||||
}
|
||||
|
||||
Block header = analysis_result.before_aggregation->getResultColumns();
|
||||
|
||||
@ -564,7 +567,8 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
|
||||
|
||||
if (options.to_stage == QueryProcessingStage::Enum::WithMergeableStateAfterAggregation)
|
||||
{
|
||||
return analysis_result.before_order_and_select->getResultColumns();
|
||||
// What's the difference with selected_columns?
|
||||
return analysis_result.before_order_by->getResultColumns();
|
||||
}
|
||||
|
||||
return analysis_result.final_projection->getResultColumns();
|
||||
@ -958,8 +962,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
|
||||
}
|
||||
else
|
||||
{
|
||||
executeExpression(query_plan, expressions.before_order_and_select, "Before ORDER BY and SELECT");
|
||||
executeExpression(query_plan, expressions.before_window, "Before window functions");
|
||||
executeWindow(query_plan);
|
||||
executeExpression(query_plan, expressions.before_order_by, "Before ORDER BY");
|
||||
executeDistinct(query_plan, true, expressions.selected_columns, true);
|
||||
}
|
||||
|
||||
@ -1005,8 +1010,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
|
||||
else if (expressions.hasHaving())
|
||||
executeHaving(query_plan, expressions.before_having);
|
||||
|
||||
executeExpression(query_plan, expressions.before_order_and_select, "Before ORDER BY and SELECT");
|
||||
executeExpression(query_plan, expressions.before_window,
|
||||
"Before window functions");
|
||||
executeWindow(query_plan);
|
||||
executeExpression(query_plan, expressions.before_order_by, "Before ORDER BY");
|
||||
executeDistinct(query_plan, true, expressions.selected_columns, true);
|
||||
|
||||
}
|
||||
@ -1029,10 +1036,23 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
|
||||
/** Optimization - if there are several sources and there is LIMIT, then first apply the preliminary LIMIT,
|
||||
* limiting the number of rows in each up to `offset + limit`.
|
||||
*/
|
||||
bool has_withfill = false;
|
||||
if (query.orderBy())
|
||||
{
|
||||
SortDescription order_descr = getSortDescription(query, *context);
|
||||
for (auto & desc : order_descr)
|
||||
if (desc.with_fill)
|
||||
{
|
||||
has_withfill = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
bool has_prelimit = false;
|
||||
if (!to_aggregation_stage &&
|
||||
query.limitLength() && !query.limit_with_ties && !hasWithTotalsInAnySubqueryInFromClause(query) &&
|
||||
!query.arrayJoinExpressionList() && !query.distinct && !expressions.hasLimitBy() && !settings.extremes)
|
||||
!query.arrayJoinExpressionList() && !query.distinct && !expressions.hasLimitBy() && !settings.extremes &&
|
||||
!has_withfill)
|
||||
{
|
||||
executePreLimit(query_plan, false);
|
||||
has_prelimit = true;
|
||||
@ -1745,6 +1765,11 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific
|
||||
|
||||
void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const ActionsDAGPtr & expression, const std::string & description)
|
||||
{
|
||||
if (!expression)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), expression);
|
||||
|
||||
expression_step->setStepDescription(description);
|
||||
|
||||
@ -90,8 +90,12 @@ std::vector<ASTs> PredicateExpressionsOptimizer::extractTablesPredicates(const A
|
||||
ExpressionInfoVisitor::Data expression_info{.context = context, .tables = tables_with_columns};
|
||||
ExpressionInfoVisitor(expression_info).visit(predicate_expression);
|
||||
|
||||
if (expression_info.is_stateful_function || !expression_info.is_deterministic_function)
|
||||
return {}; /// Not optimized when predicate contains stateful function or indeterministic function
|
||||
if (expression_info.is_stateful_function
|
||||
|| !expression_info.is_deterministic_function
|
||||
|| expression_info.is_window_function)
|
||||
{
|
||||
return {}; /// Not optimized when predicate contains stateful function or indeterministic function or window functions
|
||||
}
|
||||
|
||||
if (!expression_info.is_array_join)
|
||||
{
|
||||
@ -190,6 +194,12 @@ bool PredicateExpressionsOptimizer::tryMovePredicatesFromHavingToWhere(ASTSelect
|
||||
if (expression_info.is_stateful_function)
|
||||
return false;
|
||||
|
||||
if (expression_info.is_window_function)
|
||||
{
|
||||
// Window functions are not allowed in either HAVING or WHERE.
|
||||
return false;
|
||||
}
|
||||
|
||||
if (expression_info.is_aggregate_function)
|
||||
having_predicates.emplace_back(moving_predicate);
|
||||
else
|
||||
|
||||
@ -88,7 +88,7 @@ bool PredicateRewriteVisitorData::rewriteSubquery(ASTSelectQuery & subquery, con
|
||||
|| (!optimize_with && subquery.with())
|
||||
|| subquery.withFill()
|
||||
|| subquery.limitBy() || subquery.limitLength()
|
||||
|| hasStatefulFunction(subquery.select(), context))
|
||||
|| hasNonRewritableFunction(subquery.select(), context))
|
||||
return false;
|
||||
|
||||
for (const auto & predicate : predicates)
|
||||
|
||||
@ -148,9 +148,9 @@ void QueryNormalizer::visit(ASTSelectQuery & select, const ASTPtr &, Data & data
|
||||
/// Don't go into select query. It processes children itself.
|
||||
/// Do not go to the left argument of lambda expressions, so as not to replace the formal parameters
|
||||
/// on aliases in expressions of the form 123 AS x, arrayMap(x -> 1, [2]).
|
||||
void QueryNormalizer::visitChildren(const ASTPtr & node, Data & data)
|
||||
void QueryNormalizer::visitChildren(IAST * node, Data & data)
|
||||
{
|
||||
if (const auto * func_node = node->as<ASTFunction>())
|
||||
if (auto * func_node = node->as<ASTFunction>())
|
||||
{
|
||||
if (func_node->tryGetQueryArgument())
|
||||
{
|
||||
@ -176,6 +176,16 @@ void QueryNormalizer::visitChildren(const ASTPtr & node, Data & data)
|
||||
visit(child, data);
|
||||
}
|
||||
}
|
||||
|
||||
if (func_node->window_partition_by)
|
||||
{
|
||||
visitChildren(func_node->window_partition_by.get(), data);
|
||||
}
|
||||
|
||||
if (func_node->window_order_by)
|
||||
{
|
||||
visitChildren(func_node->window_order_by.get(), data);
|
||||
}
|
||||
}
|
||||
else if (!node->as<ASTSelectQuery>())
|
||||
{
|
||||
@ -221,7 +231,7 @@ void QueryNormalizer::visit(ASTPtr & ast, Data & data)
|
||||
if (ast.get() != initial_ast.get())
|
||||
visit(ast, data);
|
||||
else
|
||||
visitChildren(ast, data);
|
||||
visitChildren(ast.get(), data);
|
||||
|
||||
current_asts.erase(initial_ast.get());
|
||||
current_asts.erase(ast.get());
|
||||
|
||||
@ -69,7 +69,7 @@ private:
|
||||
static void visit(ASTTablesInSelectQueryElement &, const ASTPtr &, Data &);
|
||||
static void visit(ASTSelectQuery &, const ASTPtr &, Data &);
|
||||
|
||||
static void visitChildren(const ASTPtr &, Data & data);
|
||||
static void visitChildren(IAST * node, Data & data);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -29,6 +29,7 @@
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/WriteBufferFromOStream.h>
|
||||
#include <Storages/IStorage.h>
|
||||
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
@ -445,6 +446,8 @@ std::vector<const ASTFunction *> getAggregates(ASTPtr & query, const ASTSelectQu
|
||||
for (auto & arg : node->arguments->children)
|
||||
{
|
||||
assertNoAggregates(arg, "inside another aggregate function");
|
||||
// We also can't have window functions inside aggregate functions,
|
||||
// because the window functions are calculated later.
|
||||
assertNoWindows(arg, "inside an aggregate function");
|
||||
}
|
||||
}
|
||||
@ -454,7 +457,9 @@ std::vector<const ASTFunction *> getAggregates(ASTPtr & query, const ASTSelectQu
|
||||
|
||||
std::vector<const ASTFunction *> getWindowFunctions(ASTPtr & query, const ASTSelectQuery & select_query)
|
||||
{
|
||||
/// There can not be window functions inside the WHERE and PREWHERE.
|
||||
/// There can not be window functions inside the WHERE, PREWHERE and HAVING
|
||||
if (select_query.having())
|
||||
assertNoWindows(select_query.having(), "in HAVING");
|
||||
if (select_query.where())
|
||||
assertNoWindows(select_query.where(), "in WHERE");
|
||||
if (select_query.prewhere())
|
||||
@ -463,17 +468,34 @@ std::vector<const ASTFunction *> getWindowFunctions(ASTPtr & query, const ASTSel
|
||||
GetAggregatesVisitor::Data data;
|
||||
GetAggregatesVisitor(data).visit(query);
|
||||
|
||||
/// There can not be other window functions within the aggregate functions.
|
||||
/// Window functions cannot be inside aggregates or other window functions.
|
||||
/// Aggregate functions can be inside window functions because they are
|
||||
/// calculated earlier.
|
||||
for (const ASTFunction * node : data.window_functions)
|
||||
{
|
||||
if (node->arguments)
|
||||
{
|
||||
for (auto & arg : node->arguments->children)
|
||||
{
|
||||
assertNoAggregates(arg, "inside a window function");
|
||||
assertNoWindows(arg, "inside another window function");
|
||||
}
|
||||
}
|
||||
|
||||
if (node->window_partition_by)
|
||||
{
|
||||
for (auto & arg : node->window_partition_by->children)
|
||||
{
|
||||
assertNoWindows(arg, "inside PARTITION BY of a window");
|
||||
}
|
||||
}
|
||||
|
||||
if (node->window_order_by)
|
||||
{
|
||||
for (auto & arg : node->window_order_by->children)
|
||||
{
|
||||
assertNoWindows(arg, "inside ORDER BY of a window");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data.window_functions;
|
||||
|
||||
@ -39,6 +39,16 @@ void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const
|
||||
(*it)->appendColumnName(ostr);
|
||||
}
|
||||
writeChar(')', ostr);
|
||||
|
||||
if (is_window_function)
|
||||
{
|
||||
writeCString(" OVER (", ostr);
|
||||
FormatSettings settings{ostr, true /* one_line */};
|
||||
FormatState state;
|
||||
FormatStateStacked frame;
|
||||
appendWindowDescription(settings, state, frame);
|
||||
writeCString(")", ostr);
|
||||
}
|
||||
}
|
||||
|
||||
/** Get the text that identifies this element. */
|
||||
@ -57,17 +67,20 @@ ASTPtr ASTFunction::clone() const
|
||||
|
||||
if (window_name)
|
||||
{
|
||||
res->set(res->window_name, window_name->clone());
|
||||
res->window_name = window_name->clone();
|
||||
res->children.push_back(res->window_name);
|
||||
}
|
||||
|
||||
if (window_partition_by)
|
||||
{
|
||||
res->set(res->window_partition_by, window_partition_by->clone());
|
||||
res->window_partition_by = window_partition_by->clone();
|
||||
res->children.push_back(res->window_partition_by);
|
||||
}
|
||||
|
||||
if (window_order_by)
|
||||
{
|
||||
res->set(res->window_order_by, window_order_by->clone());
|
||||
res->window_order_by = window_order_by->clone();
|
||||
res->children.push_back(res->window_order_by);
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
@ -21,9 +21,25 @@ public:
|
||||
ASTPtr parameters;
|
||||
|
||||
bool is_window_function = false;
|
||||
ASTIdentifier * window_name;
|
||||
ASTExpressionList * window_partition_by;
|
||||
ASTExpressionList * window_order_by;
|
||||
|
||||
// We have to make these fields ASTPtr because this is what the visitors
|
||||
// expect. Some of them take const ASTPtr & (makes no sense), and some
|
||||
// take ASTPtr & and modify it. I don't understand how the latter is
|
||||
// compatible with also having an owning `children` array -- apparently it
|
||||
// leads to some dangling children that are not referenced by the fields of
|
||||
// the AST class itself. Some older code hints at the idea of having
|
||||
// ownership in `children` only, and making the class fields to be raw
|
||||
// pointers of proper type (see e.g. IAST::set), but this is not compatible
|
||||
// with the visitor interface.
|
||||
|
||||
// ASTIdentifier
|
||||
ASTPtr window_name;
|
||||
|
||||
// ASTExpressionList
|
||||
ASTPtr window_partition_by;
|
||||
|
||||
// ASTExpressionList of
|
||||
ASTPtr window_order_by;
|
||||
|
||||
/// do not print empty parentheses if there are no args - compatibility with new AST for data types and engine names.
|
||||
bool no_empty_args = false;
|
||||
|
||||
@ -419,7 +419,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
|
||||
ParserIdentifier window_name_parser;
|
||||
if (window_name_parser.parse(pos, window_name_ast, expected))
|
||||
{
|
||||
function->set(function->window_name, window_name_ast);
|
||||
function->children.push_back(window_name_ast);
|
||||
function->window_name = window_name_ast;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
@ -442,7 +443,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
|
||||
ASTPtr partition_by_ast;
|
||||
if (columns_partition_by.parse(pos, partition_by_ast, expected))
|
||||
{
|
||||
function->set(function->window_partition_by, partition_by_ast);
|
||||
function->children.push_back(partition_by_ast);
|
||||
function->window_partition_by = partition_by_ast;
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -455,7 +457,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
|
||||
ASTPtr order_by_ast;
|
||||
if (columns_order_by.parse(pos, order_by_ast, expected))
|
||||
{
|
||||
function->set(function->window_order_by, order_by_ast);
|
||||
function->children.push_back(order_by_ast);
|
||||
function->window_order_by = order_by_ast;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -46,6 +46,8 @@ static void doDescribeHeader(const Block & header, size_t count, IQueryPlanStep:
|
||||
|
||||
first = false;
|
||||
elem.dumpNameAndType(settings.out);
|
||||
settings.out << ": ";
|
||||
elem.dumpStructure(settings.out);
|
||||
settings.out << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
@ -247,6 +247,15 @@ static void explainStep(
|
||||
step.describeActions(settings);
|
||||
}
|
||||
|
||||
std::string debugExplainStep(const IQueryPlanStep & step)
|
||||
{
|
||||
WriteBufferFromOwnString out;
|
||||
IQueryPlanStep::FormatSettings settings{.out = out};
|
||||
QueryPlan::ExplainPlanOptions options{.actions = true};
|
||||
explainStep(step, settings, options);
|
||||
return out.str();
|
||||
}
|
||||
|
||||
void QueryPlan::explainPlan(WriteBuffer & buffer, const ExplainPlanOptions & options)
|
||||
{
|
||||
checkInitialized();
|
||||
@ -488,6 +497,7 @@ static bool tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Node *
|
||||
{
|
||||
auto & parent = parent_node->step;
|
||||
auto & child = child_node->step;
|
||||
|
||||
/// TODO: FilterStep
|
||||
auto * parent_expr = typeid_cast<ExpressionStep *>(parent.get());
|
||||
auto * child_expr = typeid_cast<ExpressionStep *>(child.get());
|
||||
|
||||
@ -97,4 +97,6 @@ private:
|
||||
std::vector<std::shared_ptr<Context>> interpreter_context;
|
||||
};
|
||||
|
||||
std::string debugExplainStep(const IQueryPlanStep & step);
|
||||
|
||||
}
|
||||
|
||||
@ -77,6 +77,11 @@ void WindowTransform::transform(Chunk & chunk)
|
||||
ws.argument_columns.clear();
|
||||
for (const auto column_index : ws.argument_column_indices)
|
||||
{
|
||||
// Aggregate functions can't work with constant columns, so we have to
|
||||
// materialize them like the Aggregator does.
|
||||
columns[column_index]
|
||||
= std::move(columns[column_index])->convertToFullColumnIfConst();
|
||||
|
||||
ws.argument_columns.push_back(columns[column_index].get());
|
||||
}
|
||||
|
||||
|
||||
@ -46,7 +46,7 @@ struct MergeTreeWriterSettings
|
||||
bool rewrite_primary_key;
|
||||
bool blocks_are_granules_size;
|
||||
|
||||
/// Used for AIO threshold comparsion
|
||||
/// Used for AIO threshold comparison
|
||||
/// FIXME currently doesn't work because WriteBufferAIO contain obscure bug(s)
|
||||
size_t estimated_size = 0;
|
||||
};
|
||||
|
||||
@ -26,7 +26,7 @@ void ReplicatedMergeTreeAltersSequence::addMetadataAlter(
|
||||
int alter_version, std::lock_guard<std::mutex> & /*state_lock*/)
|
||||
{
|
||||
/// Data alter (mutation) always added before. See ReplicatedMergeTreeQueue::pullLogsToQueue.
|
||||
/// So mutation alredy added to this sequence or doesn't exist.
|
||||
/// So mutation already added to this sequence or doesn't exist.
|
||||
if (!queue_state.count(alter_version))
|
||||
queue_state.emplace(alter_version, AlterState{.metadata_finished=false, .data_finished=true});
|
||||
else
|
||||
|
||||
@ -48,16 +48,15 @@ def dml_with_materialize_mysql_database(clickhouse_node, mysql_node, service_nam
|
||||
"/* Need ClickHouse support read mysql decimal unsigned_decimal DECIMAL(19, 10) UNSIGNED, _decimal DECIMAL(19, 10), */"
|
||||
"unsigned_float FLOAT UNSIGNED, _float FLOAT, "
|
||||
"unsigned_double DOUBLE UNSIGNED, _double DOUBLE, "
|
||||
"_varchar VARCHAR(10), _char CHAR(10), "
|
||||
"_varchar VARCHAR(10), _char CHAR(10), binary_col BINARY(8), "
|
||||
"/* Need ClickHouse support Enum('a', 'b', 'v') _enum ENUM('a', 'b', 'c'), */"
|
||||
"_date Date, _datetime DateTime, _timestamp TIMESTAMP, _bool BOOLEAN) ENGINE = InnoDB;")
|
||||
|
||||
# it already has some data
|
||||
mysql_node.query("""
|
||||
INSERT INTO test_database.test_table_1 VALUES(1, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char',
|
||||
INSERT INTO test_database.test_table_1 VALUES(1, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 'binary',
|
||||
'2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', true);
|
||||
""")
|
||||
|
||||
clickhouse_node.query(
|
||||
"CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(
|
||||
service_name))
|
||||
@ -65,51 +64,51 @@ def dml_with_materialize_mysql_database(clickhouse_node, mysql_node, service_nam
|
||||
assert "test_database" in clickhouse_node.query("SHOW DATABASES")
|
||||
|
||||
check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV",
|
||||
"1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t"
|
||||
"1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t"
|
||||
"2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n")
|
||||
|
||||
mysql_node.query("""
|
||||
INSERT INTO test_database.test_table_1 VALUES(2, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char',
|
||||
INSERT INTO test_database.test_table_1 VALUES(2, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 'binary',
|
||||
'2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', false);
|
||||
""")
|
||||
|
||||
check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV",
|
||||
"1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t"
|
||||
"1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t"
|
||||
"2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t"
|
||||
"varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t2020-01-01 00:00:00\t0\n")
|
||||
"varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t2020-01-01 00:00:00\t0\n")
|
||||
|
||||
mysql_node.query("UPDATE test_database.test_table_1 SET unsigned_tiny_int = 2 WHERE `key` = 1")
|
||||
|
||||
check_query(clickhouse_node, """
|
||||
SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int,
|
||||
small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer,
|
||||
unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char,
|
||||
unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col,
|
||||
_date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */
|
||||
_bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV
|
||||
""",
|
||||
"1\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t"
|
||||
"1\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t"
|
||||
"2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t"
|
||||
"varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t0\n")
|
||||
"varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t0\n")
|
||||
|
||||
# update primary key
|
||||
mysql_node.query("UPDATE test_database.test_table_1 SET `key` = 3 WHERE `unsigned_tiny_int` = 2")
|
||||
|
||||
check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int,"
|
||||
" small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, "
|
||||
" unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, "
|
||||
" unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, "
|
||||
" _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ "
|
||||
" _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV",
|
||||
"2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t"
|
||||
"varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t0\n3\t2\t-1\t2\t-2\t3\t-3\t"
|
||||
"4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t1\n")
|
||||
"varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t0\n3\t2\t-1\t2\t-2\t3\t-3\t"
|
||||
"4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t1\n")
|
||||
|
||||
mysql_node.query('DELETE FROM test_database.test_table_1 WHERE `key` = 2')
|
||||
check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int,"
|
||||
" small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, "
|
||||
" unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, "
|
||||
" unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, "
|
||||
" _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ "
|
||||
" _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV",
|
||||
"3\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t"
|
||||
"3\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t"
|
||||
"2020-01-01 00:00:00\t1\n")
|
||||
|
||||
mysql_node.query('DELETE FROM test_database.test_table_1 WHERE `unsigned_tiny_int` = 2')
|
||||
|
||||
@ -148,6 +148,13 @@ def test_table_function(started_cluster):
|
||||
assert node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() == '60000'
|
||||
conn.close()
|
||||
|
||||
def test_binary_type(started_cluster):
|
||||
conn = get_mysql_conn()
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute("CREATE TABLE clickhouse.binary_type (id INT PRIMARY KEY, data BINARY(16) NOT NULL)")
|
||||
table_function = "mysql('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format('binary_type')
|
||||
node1.query("INSERT INTO {} VALUES (42, 'clickhouse')".format('TABLE FUNCTION ' + table_function))
|
||||
assert node1.query("SELECT * FROM {}".format(table_function)) == '42\tclickhouse\\0\\0\\0\\0\\0\\0\n'
|
||||
|
||||
def test_enum_type(started_cluster):
|
||||
table_name = 'test_enum_type'
|
||||
|
||||
38
tests/performance/window_functions.xml
Normal file
38
tests/performance/window_functions.xml
Normal file
@ -0,0 +1,38 @@
|
||||
<test>
|
||||
<preconditions>
|
||||
<table_exists>hits_100m_single</table_exists>
|
||||
</preconditions>
|
||||
|
||||
<settings>
|
||||
<allow_experimental_window_functions>1</allow_experimental_window_functions>
|
||||
</settings>
|
||||
|
||||
<!--
|
||||
For some counters, find top 10 users by the numer of records.
|
||||
First with LIMIT BY, next with window functions.
|
||||
-->
|
||||
<query><![CDATA[
|
||||
select CounterID, UserID, count(*) user_hits
|
||||
from hits_100m_single
|
||||
where CounterID < 10000
|
||||
group by CounterID, UserID
|
||||
order by user_hits desc
|
||||
limit 10 by CounterID
|
||||
format Null
|
||||
]]></query>
|
||||
|
||||
<query><![CDATA[
|
||||
select *
|
||||
from (
|
||||
select CounterID, UserID, count(*) user_hits,
|
||||
count() over (partition by CounterID order by user_hits desc)
|
||||
user_rank
|
||||
from hits_100m_single
|
||||
where CounterID < 10000
|
||||
group by CounterID, UserID
|
||||
)
|
||||
where user_rank <= 10
|
||||
format Null
|
||||
]]></query>
|
||||
|
||||
</test>
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
echo 'DROP TABLE IF EXISTS long_insert' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @-
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
set -e
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
echo 'DROP TABLE IF EXISTS insert_fewer_columns' | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}" -d @-
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
set -o errexit
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
set -o errexit
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
set -o errexit
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
set -o errexit
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
set -o errexit
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
[ "$NO_SHELL_CONFIG" ] || . "$CURDIR"/../shell_config.sh
|
||||
|
||||
seq 1 1000 | sed -r 's/.+/CREATE TABLE IF NOT EXISTS buf_00097 (a UInt8) ENGINE = Buffer('$CLICKHOUSE_DATABASE', b, 1, 1, 1, 1, 1, 1, 1); DROP TABLE buf_00097;/' | $CLICKHOUSE_CLIENT -n
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
export NO_SHELL_CONFIG=1
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="SELECT sum(dummy) FROM remote('localhost', system, one) WHERE 1 GLOBAL IN (SELECT 1)"
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
set -o errexit
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -n --query="
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
function create {
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TABLE" -d 'IF EXISTS insert'
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
(echo 'SELECT number FROM system.numbers WHERE transform(number, ['; seq 1 100000 | tr '\n' ','; echo '0],['; seq 1 100000 | tr '\n' ','; echo '0]) = 10000000 LIMIT 1';) | $CLICKHOUSE_CLIENT --max_query_size=100000000
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1" -d @- <<< "DROP TABLE IF EXISTS test_00210"
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
CLICKHOUSE_TIMEZONE_ESCAPED=$($CLICKHOUSE_CLIENT --query="SELECT timezone()" | sed 's/[]\/$*.^+:()[]/\\&/g')
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS csv";
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=1" -d 'SELECT number FROM system.numbers LIMIT 10';
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
echo -ne '1,Hello\n2,World\n' | ${CLICKHOUSE_CURL} -sSF 'file=@-' "${CLICKHOUSE_URL}&query=SELECT+*+FROM+file&file_format=CSV&file_types=UInt8,String";
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# POST permits everything.
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS tskv";
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT 1 AS k, count() GROUP BY k WITH TOTALS";
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
echo -ne '\x50\x74\x32\xf2\x59\xe9\x8a\xdb\x37\xc6\x4a\xa7\xfb\x22\xc4\x39''\x82\x13\x00\x00\x00\x09\x00\x00\x00''\x90SELECT 1\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&decompress=1" --data-binary @-
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
echo 'DROP TABLE IF EXISTS bom' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @-
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT a' | wc -l
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'DROP TABLE IF EXISTS bad_arrays'
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
clickhouse_client_removed_host_parameter --host="${CLICKHOUSE_HOST}" --query="SELECT 1";
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS numbers";
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="SELECT 1"
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
clickhouse_client_removed_host_parameter --host="${CLICKHOUSE_HOST}" --query="SELECT * FROM ext" --format=Vertical --external --file=- --structure="s String" --name=ext --format=JSONEachRow <<< '{"s":"Hello"}'
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&add_http_cors_header=1" -H "Origin:smi2.ru" --data-binary @- <<< "SELECT 1" 2>&1 | grep -F "< Access-Control-Allow-Origin: *" | wc -l
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
set -e
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS json_noisy"
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
set -e
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CURL} -sS --local-port 1390 "${CLICKHOUSE_URL}&query_id=my_id&query=SELECT+port+FROM+system.processes+WHERE+query_id%3D%27my_id%27+ORDER+BY+elapsed+LIMIT+1"
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT --multiquery --query="SELECT 1; SELECT xyz; SELECT 2;" 2> /dev/null || true;
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
set -e
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
TABLE_HASH="cityHash64(groupArray(cityHash64(*)))"
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# We should have correct env vars from shell_config.sh to run this test
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
env TZ=UTC ${CLICKHOUSE_CLIENT} --use_client_time_zone=1 --query="SELECT toDateTime(1000000000)"
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
set -e
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
echo -ne "1\n2\n3\n" | $CLICKHOUSE_CLIENT --query="SELECT * FROM _data" --external --file=- --types=Int8;
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
echo 'one block'
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
set -e
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
echo -ne '\\tHello\t123\t\\N\n\\N\t\t2000-01-01 00:00:00\n' | ${CLICKHOUSE_LOCAL} --input-format=TabSeparated --output-format=TabSeparated --structure='s Nullable(String), x Nullable(UInt64), t Nullable(DateTime)' --query="SELECT * FROM table"
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
URL="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/"
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# We should have correct env vars from shell_config.sh to run this test
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# We should have correct env vars from shell_config.sh to run this test
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# We should have correct env vars from shell_config.sh to run this test
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# We should have correct env vars from shell_config.sh to run this test
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# We should have correct env vars from shell_config.sh to run this test
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
function perform()
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&max_block_size=5&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT max(number) FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]'
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
set -e
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
QUERY_FIELND_NUM=4
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="SELECT * FROM system.build_options" | perl -lnE 'print $1 if /(BUILD_DATE|BUILD_TYPE|CXX_COMPILER)\s+\S+/ || /(CXX_FLAGS|LINK_FLAGS|TZDATA_VERSION)/';
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user