mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 01:25:21 +00:00
Merge branch 'master' into imp-01600_parts
This commit is contained in:
Alexey Milovidov
commit
4d45d0be4f
1
.github/PULL_REQUEST_TEMPLATE.md
vendored
1
.github/PULL_REQUEST_TEMPLATE.md
vendored
@ -6,6 +6,7 @@ tests/ci/cancel_and_rerun_workflow_lambda/app.py
|
||||
-->
|
||||
### Changelog category (leave one):
|
||||
- New Feature
|
||||
- Experimental Feature
|
||||
- Improvement
|
||||
- Performance Improvement
|
||||
- Backward Incompatible Change
|
||||
|
||||
49
.github/workflows/backport_branches.yml
vendored
49
.github/workflows/backport_branches.yml
vendored
@ -159,33 +159,24 @@ jobs:
|
||||
############################################################################################
|
||||
##################################### BUILD REPORTER #######################################
|
||||
############################################################################################
|
||||
BuilderReport:
|
||||
Builds_Report:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() }}
|
||||
needs:
|
||||
- RunConfig
|
||||
- BuilderDebAarch64
|
||||
- BuilderDebAsan
|
||||
- BuilderDebDebug
|
||||
- BuilderDebRelease
|
||||
- BuilderDebTsan
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse build check
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderSpecialReport:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() }}
|
||||
needs:
|
||||
- RunConfig
|
||||
- BuilderBinDarwin
|
||||
- BuilderBinDarwinAarch64
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse special build check
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
|
||||
needs: [RunConfig, BuilderDebAarch64, BuilderDebAsan, BuilderDebDebug, BuilderDebRelease, BuilderDebTsan, BuilderBinDarwin, BuilderBinDarwinAarch64]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
- name: Download reports
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds
|
||||
- name: Builds report
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_tsan package_debug binary_darwin binary_darwin_aarch64
|
||||
- name: Set status
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --post --job-name Builds
|
||||
############################################################################################
|
||||
#################################### INSTALL PACKAGES ######################################
|
||||
############################################################################################
|
||||
@ -256,8 +247,7 @@ jobs:
|
||||
FinishCheck:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs:
|
||||
- BuilderReport
|
||||
- BuilderSpecialReport
|
||||
- Builds_Report
|
||||
- FunctionalStatelessTestAsan
|
||||
- FunctionalStatefulTestDebug
|
||||
- StressTestTsan
|
||||
@ -273,5 +263,8 @@ jobs:
|
||||
- name: Finish label
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
# update mergeable check
|
||||
python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
|
||||
# update overall ci report
|
||||
python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
|
||||
python3 merge_pr.py
|
||||
|
||||
29
.github/workflows/create_release.yml
vendored
Normal file
29
.github/workflows/create_release.yml
vendored
Normal file
@ -0,0 +1,29 @@
|
||||
name: CreateRelease
|
||||
|
||||
concurrency:
|
||||
group: release
|
||||
|
||||
'on':
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
sha:
|
||||
description: 'The SHA hash of the commit from which to create the release'
|
||||
required: true
|
||||
type: string
|
||||
type:
|
||||
description: 'The type of release: "new" for a new release or "patch" for a patch release'
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- new
|
||||
- patch
|
||||
|
||||
jobs:
|
||||
Release:
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
- name: Print greeting
|
||||
run: |
|
||||
python3 ./tests/ci/release.py --commit ${{ inputs.sha }} --type ${{ inputs.type }} --dry-run
|
||||
4
.github/workflows/master.yml
vendored
4
.github/workflows/master.yml
vendored
@ -117,11 +117,11 @@ jobs:
|
||||
# Reports should run even if Builds_1/2 fail - run them separately, not in Tests_1/2/3
|
||||
Builds_Report:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
|
||||
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
|
||||
needs: [RunConfig, Builds_1, Builds_2]
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse build check
|
||||
test_name: Builds
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
||||
|
||||
15
.github/workflows/merge_queue.yml
vendored
15
.github/workflows/merge_queue.yml
vendored
@ -96,20 +96,15 @@ jobs:
|
||||
stage: Tests_1
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
||||
################################# Stage Final #################################
|
||||
#
|
||||
FinishCheck:
|
||||
if: ${{ !cancelled() }}
|
||||
CheckReadyForMerge:
|
||||
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
|
||||
# Test_2 or Test_3 must not have jobs required for Mergeable check
|
||||
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Tests_1]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
- name: Check sync status
|
||||
- name: Check and set merge status
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 sync_pr.py --status
|
||||
- name: Finish label
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
|
||||
python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
|
||||
|
||||
4
.github/workflows/pull_request.yml
vendored
4
.github/workflows/pull_request.yml
vendored
@ -146,11 +146,11 @@ jobs:
|
||||
# Reports should run even if Builds_1/2 fail - run them separately (not in Tests_1/2/3)
|
||||
Builds_Report:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
|
||||
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
|
||||
needs: [RunConfig, StyleCheck, Builds_1, Builds_2]
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse build check
|
||||
test_name: Builds
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
||||
|
||||
51
.github/workflows/release_branches.yml
vendored
51
.github/workflows/release_branches.yml
vendored
@ -176,35 +176,24 @@ jobs:
|
||||
############################################################################################
|
||||
##################################### BUILD REPORTER #######################################
|
||||
############################################################################################
|
||||
BuilderReport:
|
||||
Builds_Report:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() }}
|
||||
needs:
|
||||
- RunConfig
|
||||
- BuilderDebRelease
|
||||
- BuilderDebAarch64
|
||||
- BuilderDebAsan
|
||||
- BuilderDebTsan
|
||||
- BuilderDebUBsan
|
||||
- BuilderDebMsan
|
||||
- BuilderDebDebug
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse build check
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderSpecialReport:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() }}
|
||||
needs:
|
||||
- RunConfig
|
||||
- BuilderBinDarwin
|
||||
- BuilderBinDarwinAarch64
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse special build check
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64, BuilderDebAsan, BuilderDebUBsan, BuilderDebMsan, BuilderDebTsan, BuilderDebDebug, BuilderBinDarwin, BuilderBinDarwinAarch64]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
- name: Download reports
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds
|
||||
- name: Builds report
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_msan package_ubsan package_tsan package_debug binary_darwin binary_darwin_aarch64
|
||||
- name: Set status
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --post --job-name Builds
|
||||
MarkReleaseReady:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs:
|
||||
@ -460,8 +449,7 @@ jobs:
|
||||
needs:
|
||||
- DockerServerImage
|
||||
- DockerKeeperImage
|
||||
- BuilderReport
|
||||
- BuilderSpecialReport
|
||||
- Builds_Report
|
||||
- MarkReleaseReady
|
||||
- FunctionalStatelessTestDebug
|
||||
- FunctionalStatelessTestRelease
|
||||
@ -496,4 +484,7 @@ jobs:
|
||||
- name: Finish label
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
# update mergeable check
|
||||
python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
|
||||
# update overall ci report
|
||||
python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
|
||||
|
||||
164
CHANGELOG.md
164
CHANGELOG.md
@ -1,4 +1,5 @@
|
||||
### Table of Contents
|
||||
**[ClickHouse release v24.6, 2024-07-01](#246)**<br/>
|
||||
**[ClickHouse release v24.5, 2024-05-30](#245)**<br/>
|
||||
**[ClickHouse release v24.4, 2024-04-30](#244)**<br/>
|
||||
**[ClickHouse release v24.3 LTS, 2024-03-26](#243)**<br/>
|
||||
@ -8,6 +9,169 @@
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### <a id="246"></a> ClickHouse release 24.6, 2024-07-01
|
||||
|
||||
#### Backward Incompatible Change
|
||||
* Enable asynchronous load of databases and tables by default. See the `async_load_databases` in config.xml. While this change is fully compatible, it can introduce a difference in behavior. When `async_load_databases` is false, as in the previous versions, the server will not accept connections until all tables are loaded. When `async_load_databases` is true, as in the new version, the server can accept connections before all the tables are loaded. If a query is made to a table that is not yet loaded, it will wait for the table's loading, which can take considerable time. It can change the behavior of the server if it is part of a large distributed system under a load balancer. In the first case, the load balancer can get a connection refusal and quickly failover to another server. In the second case, the load balancer can connect to a server that is still loading the tables, and the query will have a higher latency. Moreover, if many queries accumulate in the waiting state, it can lead to a "thundering herd" problem when they start processing simultaneously. This can make a difference only for highly loaded distributed backends. You can set the value of `async_load_databases` to false to avoid this problem. [#57695](https://github.com/ClickHouse/ClickHouse/pull/57695) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Setting `replace_long_file_name_to_hash` is enabled by default for `MergeTree` tables. [#64457](https://github.com/ClickHouse/ClickHouse/pull/64457) ([Anton Popov](https://github.com/CurtizJ)). This setting is fully compatible, and no actions needed during upgrade. The new data format is supported from all versions starting from 23.9. After enabling this setting, you can no longer downgrade to a version 23.8 or older.
|
||||
* Some invalid queries will fail earlier during parsing. Note: disabled the support for inline KQL expressions (the experimental Kusto language) when they are put into a `kql` table function without a string literal, e.g. `kql(garbage | trash)` instead of `kql('garbage | trash')` or `kql($$garbage | trash$$)`. This feature was introduced unintentionally and should not exist. [#61500](https://github.com/ClickHouse/ClickHouse/pull/61500) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Rework parallel processing in `Ordered` mode of storage `S3Queue`. This PR is backward incompatible for Ordered mode if you used settings `s3queue_processing_threads_num` or `s3queue_total_shards_num`. Setting `s3queue_total_shards_num` is deleted, previously it was allowed to use only under `s3queue_allow_experimental_sharded_mode`, which is now deprecated. A new setting is added - `s3queue_buckets`. [#64349](https://github.com/ClickHouse/ClickHouse/pull/64349) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* New functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` were added. Unlike the existing functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake`, the new functions are compatible with function `generateSnowflakeID`, i.e. they accept the snowflake IDs generated by `generateSnowflakeID` and produce snowflake IDs of the same type as `generateSnowflakeID` (i.e. `UInt64`). Furthermore, the new functions default to the UNIX epoch (aka. 1970-01-01), just like `generateSnowflakeID`. If necessary, a different epoch, e.g. Twitter's/X's epoch 2010-11-04 aka. 1288834974657 msec since UNIX epoch, can be passed. The old conversion functions are deprecated and will be removed after a transition period: to use them regardless, enable setting `allow_deprecated_snowflake_conversion_functions`. [#64948](https://github.com/ClickHouse/ClickHouse/pull/64948) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
|
||||
#### New Feature
|
||||
* Allow to store named collections in ClickHouse Keeper. [#64574](https://github.com/ClickHouse/ClickHouse/pull/64574) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Support empty tuples. [#55061](https://github.com/ClickHouse/ClickHouse/pull/55061) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Add Hilbert Curve encode and decode functions. [#60156](https://github.com/ClickHouse/ClickHouse/pull/60156) ([Artem Mustafin](https://github.com/Artemmm91)).
|
||||
* Add support for index analysis over `hilbertEncode`. [#64662](https://github.com/ClickHouse/ClickHouse/pull/64662) ([Artem Mustafin](https://github.com/Artemmm91)).
|
||||
* Added support for reading `LINESTRING` geometry in the WKT format using function `readWKTLineString`. [#62519](https://github.com/ClickHouse/ClickHouse/pull/62519) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Allow to attach parts from a different disk. [#63087](https://github.com/ClickHouse/ClickHouse/pull/63087) ([Unalian](https://github.com/Unalian)).
|
||||
* Added new SQL functions `generateSnowflakeID` for generating Twitter-style Snowflake IDs. [#63577](https://github.com/ClickHouse/ClickHouse/pull/63577) ([Danila Puzov](https://github.com/kazalika)).
|
||||
* Added `merge_workload` and `mutation_workload` settings to regulate how resources are utilized and shared between merges, mutations and other workloads. [#64061](https://github.com/ClickHouse/ClickHouse/pull/64061) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Add support for comparing `IPv4` and `IPv6` types using the `=` operator. [#64292](https://github.com/ClickHouse/ClickHouse/pull/64292) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)).
|
||||
* Support decimal arguments in binary math functions (pow, atan2, max2, min2, hypot). [#64582](https://github.com/ClickHouse/ClickHouse/pull/64582) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
|
||||
* Added SQL functions `parseReadableSize` (along with `OrNull` and `OrZero` variants). [#64742](https://github.com/ClickHouse/ClickHouse/pull/64742) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)).
|
||||
* Add server settings `max_table_num_to_throw` and `max_database_num_to_throw` to limit the number of databases or tables on `CREATE` queries. [#64781](https://github.com/ClickHouse/ClickHouse/pull/64781) ([Xu Jia](https://github.com/XuJia0210)).
|
||||
* Add `_time` virtual column to file alike storages (s3/file/hdfs/url/azureBlobStorage). [#64947](https://github.com/ClickHouse/ClickHouse/pull/64947) ([Ilya Golshtein](https://github.com/ilejn)).
|
||||
* Introduced new functions `base64URLEncode`, `base64URLDecode` and `tryBase64URLDecode`. [#64991](https://github.com/ClickHouse/ClickHouse/pull/64991) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
|
||||
* Add new function `editDistanceUTF8`, which calculates the [edit distance](https://en.wikipedia.org/wiki/Edit_distance) between two UTF8 strings. [#65269](https://github.com/ClickHouse/ClickHouse/pull/65269) ([LiuNeng](https://github.com/liuneng1994)).
|
||||
* Add `http_response_headers` setting to support custom response headers in custom HTTP handlers. [#63562](https://github.com/ClickHouse/ClickHouse/pull/63562) ([Grigorii](https://github.com/GSokol)).
|
||||
* Added a new table function `loop` to support returning query results in an infinite loop. [#63452](https://github.com/ClickHouse/ClickHouse/pull/63452) ([Sariel](https://github.com/sarielwxm)). This is useful for testing.
|
||||
* Introduced two additional columns in the `system.query_log`: `used_privileges` and `missing_privileges`. `used_privileges` is populated with the privileges that were checked during query execution, and `missing_privileges` contains required privileges that are missing. [#64597](https://github.com/ClickHouse/ClickHouse/pull/64597) ([Alexey Katsman](https://github.com/alexkats)).
|
||||
* Added a setting `output_format_pretty_display_footer_column_names` which when enabled displays column names at the end of the table for long tables (50 rows by default), with the threshold value for minimum number of rows controlled by `output_format_pretty_display_footer_column_names_min_rows`. [#65144](https://github.com/ClickHouse/ClickHouse/pull/65144) ([Shaun Struwig](https://github.com/Blargian)).
|
||||
|
||||
#### Experimental Feature
|
||||
* Introduce statistics of type "number of distinct values". [#59357](https://github.com/ClickHouse/ClickHouse/pull/59357) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Support statistics with ReplicatedMergeTree. [#64934](https://github.com/ClickHouse/ClickHouse/pull/64934) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* If "replica group" is configured for a `Replicated` database, automatically create a cluster that includes replicas from all groups. [#64312](https://github.com/ClickHouse/ClickHouse/pull/64312) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Add settings `parallel_replicas_custom_key_range_lower` and `parallel_replicas_custom_key_range_upper` to control how parallel replicas with dynamic shards parallelizes queries when using a range filter. [#64604](https://github.com/ClickHouse/ClickHouse/pull/64604) ([josh-hildred](https://github.com/josh-hildred)).
|
||||
|
||||
#### Performance Improvement
|
||||
* Add the ability to reshuffle rows during insert to optimize for size without violating the order set by `PRIMARY KEY`. It's controlled by the setting `optimize_row_order` (off by default). [#63578](https://github.com/ClickHouse/ClickHouse/pull/63578) ([Igor Markelov](https://github.com/ElderlyPassionFruit)).
|
||||
* Add a native parquet reader, which can read parquet binary to ClickHouse Columns directly. It's controlled by the setting `input_format_parquet_use_native_reader` (disabled by default). [#60361](https://github.com/ClickHouse/ClickHouse/pull/60361) ([ZhiHong Zhang](https://github.com/copperybean)).
|
||||
* Support partial trivial count optimization when the query filter is able to select exact ranges from merge tree tables. [#60463](https://github.com/ClickHouse/ClickHouse/pull/60463) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Reduce max memory usage of multithreaded `INSERT`s by collecting chunks of multiple threads in a single transform. [#61047](https://github.com/ClickHouse/ClickHouse/pull/61047) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Reduce the memory usage when using Azure object storage by using fixed memory allocation, avoiding the allocation of an extra buffer. [#63160](https://github.com/ClickHouse/ClickHouse/pull/63160) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
|
||||
* Reduce the number of virtual function calls in `ColumnNullable::size`. [#60556](https://github.com/ClickHouse/ClickHouse/pull/60556) ([HappenLee](https://github.com/HappenLee)).
|
||||
* Speedup `splitByRegexp` when the regular expression argument is a single-character. [#62696](https://github.com/ClickHouse/ClickHouse/pull/62696) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Speed up aggregation by 8-bit and 16-bit keys by keeping track of the min and max keys used. This allows to reduce the number of cells that need to be verified. [#62746](https://github.com/ClickHouse/ClickHouse/pull/62746) ([Jiebin Sun](https://github.com/jiebinn)).
|
||||
* Optimize operator IN when the left hand side is `LowCardinality` and the right is a set of constants. [#64060](https://github.com/ClickHouse/ClickHouse/pull/64060) ([Zhiguo Zhou](https://github.com/ZhiguoZh)).
|
||||
* Use a thread pool to initialize and destroy hash tables inside `ConcurrentHashJoin`. [#64241](https://github.com/ClickHouse/ClickHouse/pull/64241) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Optimized vertical merges in tables with sparse columns. [#64311](https://github.com/ClickHouse/ClickHouse/pull/64311) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Enabled prefetches of data from remote filesystem during vertical merges. It improves latency of vertical merges in tables with data stored on remote filesystem. [#64314](https://github.com/ClickHouse/ClickHouse/pull/64314) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Reduce redundant calls to `isDefault` of `ColumnSparse::filter` to improve performance. [#64426](https://github.com/ClickHouse/ClickHouse/pull/64426) ([Jiebin Sun](https://github.com/jiebinn)).
|
||||
* Speedup `find_super_nodes` and `find_big_family` keeper-client commands by making multiple asynchronous getChildren requests. [#64628](https://github.com/ClickHouse/ClickHouse/pull/64628) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Improve function `least`/`greatest` for nullable numberic type arguments. [#64668](https://github.com/ClickHouse/ClickHouse/pull/64668) ([KevinyhZou](https://github.com/KevinyhZou)).
|
||||
* Allow merging two consequent filtering steps of a query plan. This improves filter-push-down optimization if the filter condition can be pushed down from the parent step. [#64760](https://github.com/ClickHouse/ClickHouse/pull/64760) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Remove bad optimization in the vertical final implementation and re-enable vertical final algorithm by default. [#64783](https://github.com/ClickHouse/ClickHouse/pull/64783) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Remove ALIAS nodes from the filter expression. This slightly improves performance for queries with `PREWHERE` (with the new analyzer). [#64793](https://github.com/ClickHouse/ClickHouse/pull/64793) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Re-enable OpenSSL session caching. [#65111](https://github.com/ClickHouse/ClickHouse/pull/65111) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Added settings to disable materialization of skip indexes and statistics on inserts (`materialize_skip_indexes_on_insert` and `materialize_statistics_on_insert`). [#64391](https://github.com/ClickHouse/ClickHouse/pull/64391) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Use the allocated memory size to calculate the row group size and reduce the peak memory of the parquet writer in the single-threaded mode. [#64424](https://github.com/ClickHouse/ClickHouse/pull/64424) ([LiuNeng](https://github.com/liuneng1994)).
|
||||
* Improve the iterator of sparse column to reduce call of `size`. [#64497](https://github.com/ClickHouse/ClickHouse/pull/64497) ([Jiebin Sun](https://github.com/jiebinn)).
|
||||
* Update condition to use server-side copy for backups to Azure blob storage. [#64518](https://github.com/ClickHouse/ClickHouse/pull/64518) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
|
||||
* Optimized memory usage of vertical merges for tables with high number of skip indexes. [#64580](https://github.com/ClickHouse/ClickHouse/pull/64580) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Improvement
|
||||
* `SHOW CREATE TABLE` executed on top of system tables will now show the super handy comment unique for each table which will explain why this table is needed. [#63788](https://github.com/ClickHouse/ClickHouse/pull/63788) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* The second argument (scale) of functions `round()`, `roundBankers()`, `floor()`, `ceil()` and `trunc()` can now be non-const. [#64798](https://github.com/ClickHouse/ClickHouse/pull/64798) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
|
||||
* Hot reload storage policy for `Distributed` tables when adding a new disk. [#58285](https://github.com/ClickHouse/ClickHouse/pull/58285) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Avoid possible deadlock during MergeTree index analysis when scheduling threads in a saturated service. [#59427](https://github.com/ClickHouse/ClickHouse/pull/59427) ([Sean Haynes](https://github.com/seandhaynes)).
|
||||
* Several minor corner case fixes to S3 proxy support & tunneling. [#63427](https://github.com/ClickHouse/ClickHouse/pull/63427) ([Arthur Passos](https://github.com/arthurpassos)).
|
||||
* Improve io_uring resubmit visibility. Rename profile event `IOUringSQEsResubmits` -> `IOUringSQEsResubmitsAsync` and add a new one `IOUringSQEsResubmitsSync`. [#63699](https://github.com/ClickHouse/ClickHouse/pull/63699) ([Tomer Shafir](https://github.com/tomershafir)).
|
||||
* Added a new setting, `metadata_keep_free_space_bytes` to keep free space on the metadata storage disk. [#64128](https://github.com/ClickHouse/ClickHouse/pull/64128) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Add metrics to track the number of directories created and removed by the `plain_rewritable` metadata storage, and the number of entries in the local-to-remote in-memory map. [#64175](https://github.com/ClickHouse/ClickHouse/pull/64175) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* The query cache now considers identical queries with different settings as different. This increases robustness in cases where different settings (e.g. `limit` or `additional_table_filters`) would affect the query result. [#64205](https://github.com/ClickHouse/ClickHouse/pull/64205) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Support the non standard error code `QpsLimitExceeded` in object storage as a retryable error. [#64225](https://github.com/ClickHouse/ClickHouse/pull/64225) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Forbid converting a MergeTree table to replicated if the zookeeper path for this table already exists. [#64244](https://github.com/ClickHouse/ClickHouse/pull/64244) ([Kirill](https://github.com/kirillgarbar)).
|
||||
* Added a new setting `input_format_parquet_prefer_block_bytes` to control the average output block bytes, and modified the default value of `input_format_parquet_max_block_size` to 65409. [#64427](https://github.com/ClickHouse/ClickHouse/pull/64427) ([LiuNeng](https://github.com/liuneng1994)).
|
||||
* Allow proxy to be bypassed for hosts specified in `no_proxy` env variable and ClickHouse proxy configuration. [#63314](https://github.com/ClickHouse/ClickHouse/pull/63314) ([Arthur Passos](https://github.com/arthurpassos)).
|
||||
* Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Settings from the user's config don't affect merges and mutations for `MergeTree` on top of object storage. [#64456](https://github.com/ClickHouse/ClickHouse/pull/64456) ([alesapin](https://github.com/alesapin)).
|
||||
* Support the non standard error code `TotalQpsLimitExceeded` in object storage as a retryable error. [#64520](https://github.com/ClickHouse/ClickHouse/pull/64520) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Updated Advanced Dashboard for both open-source and ClickHouse Cloud versions to include a chart for 'Maximum concurrent network connections'. [#64610](https://github.com/ClickHouse/ClickHouse/pull/64610) ([Thom O'Connor](https://github.com/thomoco)).
|
||||
* Improve progress report on `zeros_mt` and `generateRandom`. [#64804](https://github.com/ClickHouse/ClickHouse/pull/64804) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Add an asynchronous metric `jemalloc.profile.active` to show whether sampling is currently active. This is an activation mechanism in addition to prof.active; both must be active for the calling thread to sample. [#64842](https://github.com/ClickHouse/ClickHouse/pull/64842) ([Unalian](https://github.com/Unalian)).
|
||||
* Remove mark of `allow_experimental_join_condition` as important. This mark may have prevented distributed queries in a mixed versions cluster from being executed successfully. [#65008](https://github.com/ClickHouse/ClickHouse/pull/65008) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Added server Asynchronous metrics `DiskGetObjectThrottler*` and `DiskGetObjectThrottler*` reflecting request per second rate limit defined with `s3_max_get_rps` and `s3_max_put_rps` disk settings and currently available number of requests that could be sent without hitting throttling limit on the disk. Metrics are defined for every disk that has a configured limit. [#65050](https://github.com/ClickHouse/ClickHouse/pull/65050) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Initialize global trace collector for `Poco::ThreadPool` (needed for Keeper, etc). [#65239](https://github.com/ClickHouse/ClickHouse/pull/65239) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Add a validation when creating a user with `bcrypt_hash`. [#65242](https://github.com/ClickHouse/ClickHouse/pull/65242) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Add profile events for number of rows read during/after `PREWHERE`. [#64198](https://github.com/ClickHouse/ClickHouse/pull/64198) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Print query in `EXPLAIN PLAN` with parallel replicas. [#64298](https://github.com/ClickHouse/ClickHouse/pull/64298) ([vdimir](https://github.com/vdimir)).
|
||||
* Rename `allow_deprecated_functions` to `allow_deprecated_error_prone_window_functions`. [#64358](https://github.com/ClickHouse/ClickHouse/pull/64358) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Respect `max_read_buffer_size` setting for file descriptors as well in the `file` table function. [#64532](https://github.com/ClickHouse/ClickHouse/pull/64532) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Disable transactions for unsupported storages even for materialized views. [#64918](https://github.com/ClickHouse/ClickHouse/pull/64918) ([alesapin](https://github.com/alesapin)).
|
||||
* Forbid `QUALIFY` clause in the old analyzer. The old analyzer ignored `QUALIFY`, so it could lead to unexpected data removal in mutations. [#65356](https://github.com/ClickHouse/ClickHouse/pull/65356) ([Dmitry Novik](https://github.com/novikd)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* A bug in Apache ORC library was fixed: Fixed ORC statistics calculation, when writing, for unsigned types on all platforms and Int8 on ARM. [#64563](https://github.com/ClickHouse/ClickHouse/pull/64563) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Returned back the behaviour of how ClickHouse works and interprets Tuples in CSV format. This change effectively reverts https://github.com/ClickHouse/ClickHouse/pull/60994 and makes it available only under a few settings: `output_format_csv_serialize_tuple_into_separate_columns`, `input_format_csv_deserialize_separate_columns_into_tuple` and `input_format_csv_try_infer_strings_from_quoted_tuples`. [#65170](https://github.com/ClickHouse/ClickHouse/pull/65170) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix a permission error where a user in a specific situation can escalate their privileges on the default database without necessary grants. [#64769](https://github.com/ClickHouse/ClickHouse/pull/64769) ([pufit](https://github.com/pufit)).
|
||||
* Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Use correct memory alignment for Distinct combinator. Previously, crash could happen because of invalid memory allocation when the combinator was used. [#65379](https://github.com/ClickHouse/ClickHouse/pull/65379) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix crash with `DISTINCT` and window functions. [#64767](https://github.com/ClickHouse/ClickHouse/pull/64767) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fixed 'set' skip index not working with IN and indexHint(). [#62083](https://github.com/ClickHouse/ClickHouse/pull/62083) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Support executing function during assignment of parameterized view value. [#63502](https://github.com/ClickHouse/ClickHouse/pull/63502) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
|
||||
* Fixed parquet memory tracking. [#63584](https://github.com/ClickHouse/ClickHouse/pull/63584) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fixed reading of columns of type `Tuple(Map(LowCardinality(String), String), ...)`. [#63956](https://github.com/ClickHouse/ClickHouse/pull/63956) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix an `Cyclic aliases` error for cyclic aliases of different type (expression and function). [#63993](https://github.com/ClickHouse/ClickHouse/pull/63993) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* This fix will use a proper redefined context with the correct definer for each individual view in the query pipeline. [#64079](https://github.com/ClickHouse/ClickHouse/pull/64079) ([pufit](https://github.com/pufit)).
|
||||
* Fix analyzer: "Not found column" error is fixed when using INTERPOLATE. [#64096](https://github.com/ClickHouse/ClickHouse/pull/64096) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Fix creating backups to S3 buckets with different credentials from the disk containing the file. [#64153](https://github.com/ClickHouse/ClickHouse/pull/64153) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* The query cache now considers two identical queries against different databases as different. The previous behavior could be used to bypass missing privileges to read from a table. [#64199](https://github.com/ClickHouse/ClickHouse/pull/64199) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix possible abort on uncaught exception in ~WriteBufferFromFileDescriptor in StatusFile. [#64206](https://github.com/ClickHouse/ClickHouse/pull/64206) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix `duplicate alias` error for distributed queries with `ARRAY JOIN`. [#64226](https://github.com/ClickHouse/ClickHouse/pull/64226) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix unexpected accurateCast from string to integer. [#64255](https://github.com/ClickHouse/ClickHouse/pull/64255) ([wudidapaopao](https://github.com/wudidapaopao)).
|
||||
* Fixed CNF simplification, in case any OR group contains mutually exclusive atoms. [#64256](https://github.com/ClickHouse/ClickHouse/pull/64256) ([Eduard Karacharov](https://github.com/korowa)).
|
||||
* Fix Query Tree size validation. [#64377](https://github.com/ClickHouse/ClickHouse/pull/64377) ([Dmitry Novik](https://github.com/novikd)).
|
||||
* Fix `Logical error: Bad cast` for `Buffer` table with `PREWHERE`. [#64388](https://github.com/ClickHouse/ClickHouse/pull/64388) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Prevent recursive logging in `blob_storage_log` when it's stored on object storage. [#64393](https://github.com/ClickHouse/ClickHouse/pull/64393) ([vdimir](https://github.com/vdimir)).
|
||||
* Fixed `CREATE TABLE AS` queries for tables with default expressions. [#64455](https://github.com/ClickHouse/ClickHouse/pull/64455) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fixed `optimize_read_in_order` behaviour for ORDER BY ... NULLS FIRST / LAST on tables with nullable keys. [#64483](https://github.com/ClickHouse/ClickHouse/pull/64483) ([Eduard Karacharov](https://github.com/korowa)).
|
||||
* Fix the `Expression nodes list expected 1 projection names` and `Unknown expression or identifier` errors for queries with aliases to `GLOBAL IN.`. [#64517](https://github.com/ClickHouse/ClickHouse/pull/64517) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix an error `Cannot find column` in distributed queries with constant CTE in the `GROUP BY` key. [#64519](https://github.com/ClickHouse/ClickHouse/pull/64519) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix the crash loop when restoring from backup is blocked by creating an MV with a definer that hasn't been restored yet. [#64595](https://github.com/ClickHouse/ClickHouse/pull/64595) ([pufit](https://github.com/pufit)).
|
||||
* Fix the output of function `formatDateTimeInJodaSyntax` when a formatter generates an uneven number of characters and the last character is `0`. For example, `SELECT formatDateTimeInJodaSyntax(toDate('2012-05-29'), 'D')` now correctly returns `150` instead of previously `15`. [#64614](https://github.com/ClickHouse/ClickHouse/pull/64614) ([LiuNeng](https://github.com/liuneng1994)).
|
||||
* Do not rewrite aggregation if `-If` combinator is already used. [#64638](https://github.com/ClickHouse/ClickHouse/pull/64638) ([Dmitry Novik](https://github.com/novikd)).
|
||||
* Fix type inference for float (in case of small buffer, i.e. `--max_read_buffer_size 1`). [#64641](https://github.com/ClickHouse/ClickHouse/pull/64641) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix bug which could lead to non-working TTLs with expressions. [#64694](https://github.com/ClickHouse/ClickHouse/pull/64694) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix removing the `WHERE` and `PREWHERE` expressions, which are always true (for the new analyzer). [#64695](https://github.com/ClickHouse/ClickHouse/pull/64695) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fixed excessive part elimination by token-based text indexes (`ngrambf` , `full_text`) when filtering by result of `startsWith`, `endsWith`, `match`, `multiSearchAny`. [#64720](https://github.com/ClickHouse/ClickHouse/pull/64720) ([Eduard Karacharov](https://github.com/korowa)).
|
||||
* Fixes incorrect behaviour of ANSI CSI escaping in the `UTF8::computeWidth` function. [#64756](https://github.com/ClickHouse/ClickHouse/pull/64756) ([Shaun Struwig](https://github.com/Blargian)).
|
||||
* Fix a case of incorrect removal of `ORDER BY` / `LIMIT BY` across subqueries. [#64766](https://github.com/ClickHouse/ClickHouse/pull/64766) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix (experimental) unequal join with subqueries for sets which are in the mixed join conditions. [#64775](https://github.com/ClickHouse/ClickHouse/pull/64775) ([lgbo](https://github.com/lgbo-ustc)).
|
||||
* Fix crash in a local cache over `plain_rewritable` disk. [#64778](https://github.com/ClickHouse/ClickHouse/pull/64778) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Keeper fix: return correct value for `zk_latest_snapshot_size` in `mntr` command. [#64784](https://github.com/ClickHouse/ClickHouse/pull/64784) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix `Cannot find column` in distributed query with `ARRAY JOIN` by `Nested` column. Fixes [#64755](https://github.com/ClickHouse/ClickHouse/issues/64755). [#64801](https://github.com/ClickHouse/ClickHouse/pull/64801) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix memory leak in slru cache policy. [#64803](https://github.com/ClickHouse/ClickHouse/pull/64803) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fixed possible incorrect memory tracking in several kinds of queries: queries that read any data from S3, queries via http protocol, asynchronous inserts. [#64844](https://github.com/ClickHouse/ClickHouse/pull/64844) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix the `Block structure mismatch` error for queries reading with `PREWHERE` from the materialized view when the materialized view has columns of different types than the source table. Fixes [#64611](https://github.com/ClickHouse/ClickHouse/issues/64611). [#64855](https://github.com/ClickHouse/ClickHouse/pull/64855) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix rare crash when table has TTL with subquery + database replicated + parallel replicas + analyzer. It's really rare, but please don't use TTLs with subqueries. [#64858](https://github.com/ClickHouse/ClickHouse/pull/64858) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix duplicating `Delete` events in `blob_storage_log` in case of large batch to delete. [#64924](https://github.com/ClickHouse/ClickHouse/pull/64924) ([vdimir](https://github.com/vdimir)).
|
||||
* Fixed `Session moved to another server` error from [Zoo]Keeper that might happen after server startup when the config has includes from [Zoo]Keeper. [#64986](https://github.com/ClickHouse/ClickHouse/pull/64986) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Fix `ALTER MODIFY COMMENT` query that was broken for parameterized VIEWs in https://github.com/ClickHouse/ClickHouse/pull/54211. [#65031](https://github.com/ClickHouse/ClickHouse/pull/65031) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Fix `host_id` in DatabaseReplicated when `cluster_secure_connection` parameter is enabled. Previously all the connections within the cluster created by DatabaseReplicated were not secure, even if the parameter was enabled. [#65054](https://github.com/ClickHouse/ClickHouse/pull/65054) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Fixing the `Not-ready Set` error after the `PREWHERE` optimization for StorageMerge. [#65057](https://github.com/ClickHouse/ClickHouse/pull/65057) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Avoid writing to finalized buffer in File-like storages. [#65063](https://github.com/ClickHouse/ClickHouse/pull/65063) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix possible infinite query duration in case of cyclic aliases. Fixes [#64849](https://github.com/ClickHouse/ClickHouse/issues/64849). [#65081](https://github.com/ClickHouse/ClickHouse/pull/65081) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix the `Unknown expression identifier` error for remote queries with `INTERPOLATE (alias)` (new analyzer). Fixes [#64636](https://github.com/ClickHouse/ClickHouse/issues/64636). [#65090](https://github.com/ClickHouse/ClickHouse/pull/65090) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix pushing arithmetic operations out of aggregation. In the new analyzer, optimization was applied only once. [#65104](https://github.com/ClickHouse/ClickHouse/pull/65104) ([Dmitry Novik](https://github.com/novikd)).
|
||||
* Fix aggregate function name rewriting in the new analyzer. [#65110](https://github.com/ClickHouse/ClickHouse/pull/65110) ([Dmitry Novik](https://github.com/novikd)).
|
||||
* Respond with 5xx instead of 200 OK in case of receive timeout while reading (parts of) the request body from the client socket. [#65118](https://github.com/ClickHouse/ClickHouse/pull/65118) ([Julian Maicher](https://github.com/jmaicher)).
|
||||
* Fix possible crash for hedged requests. [#65206](https://github.com/ClickHouse/ClickHouse/pull/65206) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix the bug in Hashed and Hashed_Array dictionary short circuit evaluation, which may read uninitialized number, leading to various errors. [#65256](https://github.com/ClickHouse/ClickHouse/pull/65256) ([jsc0218](https://github.com/jsc0218)).
|
||||
* This PR ensures that the type of the constant(IN operator's second parameter) is always visible during the IN operator's type conversion process. Otherwise, losing type information may cause some conversions to fail, such as the conversion from DateTime to Date. This fixes ([#64487](https://github.com/ClickHouse/ClickHouse/issues/64487)). [#65315](https://github.com/ClickHouse/ClickHouse/pull/65315) ([pn](https://github.com/chloro-pn)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Add support for LLVM XRay. [#64592](https://github.com/ClickHouse/ClickHouse/pull/64592) [#64837](https://github.com/ClickHouse/ClickHouse/pull/64837) ([Tomer Shafir](https://github.com/tomershafir)).
|
||||
* Unite s3/hdfs/azure storage implementations into a single class working with IObjectStorage. Same for *Cluster, data lakes and Queue storages. [#59767](https://github.com/ClickHouse/ClickHouse/pull/59767) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Refactor data part writer to remove dependencies on MergeTreeData and DataPart. [#63620](https://github.com/ClickHouse/ClickHouse/pull/63620) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Refactor `KeyCondition` and key analysis to improve PartitionPruner and trivial count optimization. This is separated from [#60463](https://github.com/ClickHouse/ClickHouse/issues/60463) . [#61459](https://github.com/ClickHouse/ClickHouse/pull/61459) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Introduce assertions to verify all functions are called with columns of the right size. [#63723](https://github.com/ClickHouse/ClickHouse/pull/63723) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Make `network` service be required when using the `rc` init script to start the ClickHouse server daemon. [#60650](https://github.com/ClickHouse/ClickHouse/pull/60650) ([Chun-Sheng, Li](https://github.com/peter279k)).
|
||||
* Reduce the size of some slow tests. [#64387](https://github.com/ClickHouse/ClickHouse/pull/64387) [#64452](https://github.com/ClickHouse/ClickHouse/pull/64452) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Replay ZooKeeper logs using keeper-bench. [#62481](https://github.com/ClickHouse/ClickHouse/pull/62481) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
|
||||
### <a id="245"></a> ClickHouse release 24.5, 2024-05-30
|
||||
|
||||
#### Backward Incompatible Change
|
||||
|
||||
@ -14,6 +14,7 @@ The following versions of ClickHouse server are currently supported with securit
|
||||
|
||||
| Version | Supported |
|
||||
|:-|:-|
|
||||
| 24.6 | ✔️ |
|
||||
| 24.5 | ✔️ |
|
||||
| 24.4 | ✔️ |
|
||||
| 24.3 | ✔️ |
|
||||
|
||||
@ -6,6 +6,9 @@ namespace
|
||||
{
|
||||
std::string getFQDNOrHostNameImpl()
|
||||
{
|
||||
#if defined(OS_DARWIN)
|
||||
return Poco::Net::DNS::hostName();
|
||||
#else
|
||||
try
|
||||
{
|
||||
return Poco::Net::DNS::thisHost().name();
|
||||
@ -14,6 +17,7 @@ namespace
|
||||
{
|
||||
return Poco::Net::DNS::hostName();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
#ifndef NetSSL_SSLManager_INCLUDED
|
||||
#define NetSSL_SSLManager_INCLUDED
|
||||
|
||||
#include <unordered_map>
|
||||
|
||||
#include <openssl/ssl.h>
|
||||
#include "Poco/BasicEvent.h"
|
||||
@ -219,6 +220,13 @@ namespace Net
|
||||
/// Unless initializeClient() has been called, the first call to this method initializes the default Context
|
||||
/// from the application configuration.
|
||||
|
||||
Context::Ptr getCustomServerContext(const std::string & name);
|
||||
/// Return custom Context used by the server.
|
||||
|
||||
Context::Ptr setCustomServerContext(const std::string & name, Context::Ptr ctx);
|
||||
/// Set custom Context used by the server.
|
||||
/// Return pointer on inserted Context or on old Context if exists.
|
||||
|
||||
PrivateKeyPassphraseHandlerPtr serverPassphraseHandler();
|
||||
/// Returns the configured passphrase handler of the server. If none is set, the method will create a default one
|
||||
/// from an application configuration.
|
||||
@ -258,6 +266,40 @@ namespace Net
|
||||
static const std::string CFG_SERVER_PREFIX;
|
||||
static const std::string CFG_CLIENT_PREFIX;
|
||||
|
||||
static const std::string CFG_PRIV_KEY_FILE;
|
||||
static const std::string CFG_CERTIFICATE_FILE;
|
||||
static const std::string CFG_CA_LOCATION;
|
||||
static const std::string CFG_VER_MODE;
|
||||
static const Context::VerificationMode VAL_VER_MODE;
|
||||
static const std::string CFG_VER_DEPTH;
|
||||
static const int VAL_VER_DEPTH;
|
||||
static const std::string CFG_ENABLE_DEFAULT_CA;
|
||||
static const bool VAL_ENABLE_DEFAULT_CA;
|
||||
static const std::string CFG_CIPHER_LIST;
|
||||
static const std::string CFG_CYPHER_LIST; // for backwards compatibility
|
||||
static const std::string VAL_CIPHER_LIST;
|
||||
static const std::string CFG_PREFER_SERVER_CIPHERS;
|
||||
static const std::string CFG_DELEGATE_HANDLER;
|
||||
static const std::string VAL_DELEGATE_HANDLER;
|
||||
static const std::string CFG_CERTIFICATE_HANDLER;
|
||||
static const std::string VAL_CERTIFICATE_HANDLER;
|
||||
static const std::string CFG_CACHE_SESSIONS;
|
||||
static const std::string CFG_SESSION_ID_CONTEXT;
|
||||
static const std::string CFG_SESSION_CACHE_SIZE;
|
||||
static const std::string CFG_SESSION_TIMEOUT;
|
||||
static const std::string CFG_EXTENDED_VERIFICATION;
|
||||
static const std::string CFG_REQUIRE_TLSV1;
|
||||
static const std::string CFG_REQUIRE_TLSV1_1;
|
||||
static const std::string CFG_REQUIRE_TLSV1_2;
|
||||
static const std::string CFG_DISABLE_PROTOCOLS;
|
||||
static const std::string CFG_DH_PARAMS_FILE;
|
||||
static const std::string CFG_ECDH_CURVE;
|
||||
|
||||
#ifdef OPENSSL_FIPS
|
||||
static const std::string CFG_FIPS_MODE;
|
||||
static const bool VAL_FIPS_MODE;
|
||||
#endif
|
||||
|
||||
protected:
|
||||
static int verifyClientCallback(int ok, X509_STORE_CTX * pStore);
|
||||
/// The return value of this method defines how errors in
|
||||
@ -314,39 +356,7 @@ namespace Net
|
||||
InvalidCertificateHandlerPtr _ptrClientCertificateHandler;
|
||||
Poco::FastMutex _mutex;
|
||||
|
||||
static const std::string CFG_PRIV_KEY_FILE;
|
||||
static const std::string CFG_CERTIFICATE_FILE;
|
||||
static const std::string CFG_CA_LOCATION;
|
||||
static const std::string CFG_VER_MODE;
|
||||
static const Context::VerificationMode VAL_VER_MODE;
|
||||
static const std::string CFG_VER_DEPTH;
|
||||
static const int VAL_VER_DEPTH;
|
||||
static const std::string CFG_ENABLE_DEFAULT_CA;
|
||||
static const bool VAL_ENABLE_DEFAULT_CA;
|
||||
static const std::string CFG_CIPHER_LIST;
|
||||
static const std::string CFG_CYPHER_LIST; // for backwards compatibility
|
||||
static const std::string VAL_CIPHER_LIST;
|
||||
static const std::string CFG_PREFER_SERVER_CIPHERS;
|
||||
static const std::string CFG_DELEGATE_HANDLER;
|
||||
static const std::string VAL_DELEGATE_HANDLER;
|
||||
static const std::string CFG_CERTIFICATE_HANDLER;
|
||||
static const std::string VAL_CERTIFICATE_HANDLER;
|
||||
static const std::string CFG_CACHE_SESSIONS;
|
||||
static const std::string CFG_SESSION_ID_CONTEXT;
|
||||
static const std::string CFG_SESSION_CACHE_SIZE;
|
||||
static const std::string CFG_SESSION_TIMEOUT;
|
||||
static const std::string CFG_EXTENDED_VERIFICATION;
|
||||
static const std::string CFG_REQUIRE_TLSV1;
|
||||
static const std::string CFG_REQUIRE_TLSV1_1;
|
||||
static const std::string CFG_REQUIRE_TLSV1_2;
|
||||
static const std::string CFG_DISABLE_PROTOCOLS;
|
||||
static const std::string CFG_DH_PARAMS_FILE;
|
||||
static const std::string CFG_ECDH_CURVE;
|
||||
|
||||
#ifdef OPENSSL_FIPS
|
||||
static const std::string CFG_FIPS_MODE;
|
||||
static const bool VAL_FIPS_MODE;
|
||||
#endif
|
||||
std::unordered_map<std::string, Context::Ptr> _mapPtrServerContexts;
|
||||
|
||||
friend class Poco::SingletonHolder<SSLManager>;
|
||||
friend class Context;
|
||||
|
||||
@ -428,6 +428,23 @@ void SSLManager::initCertificateHandler(bool server)
|
||||
}
|
||||
|
||||
|
||||
Context::Ptr SSLManager::getCustomServerContext(const std::string & name)
|
||||
{
|
||||
Poco::FastMutex::ScopedLock lock(_mutex);
|
||||
auto it = _mapPtrServerContexts.find(name);
|
||||
if (it != _mapPtrServerContexts.end())
|
||||
return it->second;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Context::Ptr SSLManager::setCustomServerContext(const std::string & name, Context::Ptr ctx)
|
||||
{
|
||||
Poco::FastMutex::ScopedLock lock(_mutex);
|
||||
ctx = _mapPtrServerContexts.insert({name, ctx}).first->second;
|
||||
return ctx;
|
||||
}
|
||||
|
||||
|
||||
Poco::Util::AbstractConfiguration& SSLManager::appConfig()
|
||||
{
|
||||
try
|
||||
|
||||
@ -34,7 +34,7 @@ if (OS_LINUX)
|
||||
# avoid spurious latencies and additional work associated with
|
||||
# MADV_DONTNEED. See
|
||||
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
|
||||
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
|
||||
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
|
||||
else()
|
||||
set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
|
||||
endif()
|
||||
|
||||
2
contrib/mariadb-connector-c
vendored
2
contrib/mariadb-connector-c
vendored
@ -1 +1 @@
|
||||
Subproject commit e39608998f5f6944ece9ec61f48e9172ec1de660
|
||||
Subproject commit d0a788c5b9fcaca2368d9233770d3ca91ea79f88
|
||||
@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.5.3.5"
|
||||
ARG VERSION="24.6.1.4423"
|
||||
ARG PACKAGES="clickhouse-keeper"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.5.3.5"
|
||||
ARG VERSION="24.6.1.4423"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
|
||||
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
|
||||
ARG VERSION="24.5.3.5"
|
||||
ARG VERSION="24.6.1.4423"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
|
||||
#docker-official-library:off
|
||||
|
||||
@ -37,6 +37,7 @@ RUN pip3 install \
|
||||
tqdm==4.66.4 \
|
||||
types-requests \
|
||||
unidiff \
|
||||
jwt \
|
||||
&& rm -rf /root/.cache/pip
|
||||
|
||||
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8
|
||||
|
||||
30
docs/changelogs/v24.4.3.25-stable.md
Normal file
30
docs/changelogs/v24.4.3.25-stable.md
Normal file
@ -0,0 +1,30 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.4.3.25-stable (a915dd4eda4) FIXME as compared to v24.4.2.141-stable (9e23d27bd11)
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#65130](https://github.com/ClickHouse/ClickHouse/issues/65130): Decrease the `unit-test` image a few times. [#65102](https://github.com/ClickHouse/ClickHouse/pull/65102) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
|
||||
* Backported in [#64982](https://github.com/ClickHouse/ClickHouse/issues/64982): Fix the `Block structure mismatch` error for queries reading with `PREWHERE` from the materialized view when the materialized view has columns of different types than the source table. Fixes [#64611](https://github.com/ClickHouse/ClickHouse/issues/64611). [#64855](https://github.com/ClickHouse/ClickHouse/pull/64855) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#64974](https://github.com/ClickHouse/ClickHouse/issues/64974): Fix rare crash when table has TTL with subquery + database replicated + parallel replicas + analyzer. It's really rare, but please don't use TTLs with subqueries. [#64858](https://github.com/ClickHouse/ClickHouse/pull/64858) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#65072](https://github.com/ClickHouse/ClickHouse/issues/65072): Fix `ALTER MODIFY COMMENT` query that was broken for parameterized VIEWs in https://github.com/ClickHouse/ClickHouse/pull/54211. [#65031](https://github.com/ClickHouse/ClickHouse/pull/65031) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Backported in [#65177](https://github.com/ClickHouse/ClickHouse/issues/65177): Fix the `Unknown expression identifier` error for remote queries with `INTERPOLATE (alias)` (new analyzer). Fixes [#64636](https://github.com/ClickHouse/ClickHouse/issues/64636). [#65090](https://github.com/ClickHouse/ClickHouse/pull/65090) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#65263](https://github.com/ClickHouse/ClickHouse/issues/65263): Fix the bug in Hashed and Hashed_Array dictionary short circuit evaluation, which may read uninitialized number, leading to various errors. [#65256](https://github.com/ClickHouse/ClickHouse/pull/65256) ([jsc0218](https://github.com/jsc0218)).
|
||||
|
||||
#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
|
||||
|
||||
* Backported in [#65285](https://github.com/ClickHouse/ClickHouse/issues/65285): Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#65114](https://github.com/ClickHouse/ClickHouse/issues/65114): Adjust the `version_helper` and script to a new release scheme. [#64759](https://github.com/ClickHouse/ClickHouse/pull/64759) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#65225](https://github.com/ClickHouse/ClickHouse/issues/65225): Capture weak_ptr of ContextAccess for safety. [#65051](https://github.com/ClickHouse/ClickHouse/pull/65051) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Backported in [#65217](https://github.com/ClickHouse/ClickHouse/issues/65217): Fix false positives leaky memory warnings in OpenSSL. [#65125](https://github.com/ClickHouse/ClickHouse/pull/65125) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
|
||||
41
docs/changelogs/v24.5.4.49-stable.md
Normal file
41
docs/changelogs/v24.5.4.49-stable.md
Normal file
@ -0,0 +1,41 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.5.4.49-stable (63b760955a0) FIXME as compared to v24.5.3.5-stable (e0eb66f8e17)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#65886](https://github.com/ClickHouse/ClickHouse/issues/65886): Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Backported in [#65304](https://github.com/ClickHouse/ClickHouse/issues/65304): Returned back the behaviour of how ClickHouse works and interprets Tuples in CSV format. This change effectively reverts https://github.com/ClickHouse/ClickHouse/pull/60994 and makes it available only under a few settings: `output_format_csv_serialize_tuple_into_separate_columns`, `input_format_csv_deserialize_separate_columns_into_tuple` and `input_format_csv_try_infer_strings_from_quoted_tuples`. [#65170](https://github.com/ClickHouse/ClickHouse/pull/65170) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Backported in [#65896](https://github.com/ClickHouse/ClickHouse/issues/65896): Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
|
||||
#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
|
||||
* Backported in [#65287](https://github.com/ClickHouse/ClickHouse/issues/65287): Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#65374](https://github.com/ClickHouse/ClickHouse/issues/65374): Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Backported in [#65437](https://github.com/ClickHouse/ClickHouse/issues/65437): Forbid `QUALIFY` clause in the old analyzer. The old analyzer ignored `QUALIFY`, so it could lead to unexpected data removal in mutations. [#65356](https://github.com/ClickHouse/ClickHouse/pull/65356) ([Dmitry Novik](https://github.com/novikd)).
|
||||
* Backported in [#65450](https://github.com/ClickHouse/ClickHouse/issues/65450): Use correct memory alignment for Distinct combinator. Previously, crash could happen because of invalid memory allocation when the combinator was used. [#65379](https://github.com/ClickHouse/ClickHouse/pull/65379) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#65712](https://github.com/ClickHouse/ClickHouse/issues/65712): Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#65681](https://github.com/ClickHouse/ClickHouse/issues/65681): Fix `duplicate alias` error for distributed queries with `ARRAY JOIN`. [#64226](https://github.com/ClickHouse/ClickHouse/pull/64226) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#65331](https://github.com/ClickHouse/ClickHouse/issues/65331): Fix the crash loop when restoring from backup is blocked by creating an MV with a definer that hasn't been restored yet. [#64595](https://github.com/ClickHouse/ClickHouse/pull/64595) ([pufit](https://github.com/pufit)).
|
||||
* Backported in [#64835](https://github.com/ClickHouse/ClickHouse/issues/64835): Fix bug which could lead to non-working TTLs with expressions. [#64694](https://github.com/ClickHouse/ClickHouse/pull/64694) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#65542](https://github.com/ClickHouse/ClickHouse/issues/65542): Fix crash for `ALTER TABLE ... ON CLUSTER ... MODIFY SQL SECURITY`. [#64957](https://github.com/ClickHouse/ClickHouse/pull/64957) ([pufit](https://github.com/pufit)).
|
||||
* Backported in [#65580](https://github.com/ClickHouse/ClickHouse/issues/65580): Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#65618](https://github.com/ClickHouse/ClickHouse/issues/65618): Fix possible infinite query duration in case of cyclic aliases. Fixes [#64849](https://github.com/ClickHouse/ClickHouse/issues/64849). [#65081](https://github.com/ClickHouse/ClickHouse/pull/65081) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#65617](https://github.com/ClickHouse/ClickHouse/issues/65617): Fix aggregate function name rewriting in the new analyzer. [#65110](https://github.com/ClickHouse/ClickHouse/pull/65110) ([Dmitry Novik](https://github.com/novikd)).
|
||||
* Backported in [#65732](https://github.com/ClickHouse/ClickHouse/issues/65732): Eliminate injective function in argument of functions `uniq*` recursively. This used to work correctly but was broken in the new analyzer. [#65140](https://github.com/ClickHouse/ClickHouse/pull/65140) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Backported in [#65265](https://github.com/ClickHouse/ClickHouse/issues/65265): Fix the bug in Hashed and Hashed_Array dictionary short circuit evaluation, which may read uninitialized number, leading to various errors. [#65256](https://github.com/ClickHouse/ClickHouse/pull/65256) ([jsc0218](https://github.com/jsc0218)).
|
||||
* Backported in [#65663](https://github.com/ClickHouse/ClickHouse/issues/65663): Disable `non-intersecting-parts` optimization for queries with `FINAL` in case of `read-in-order` optimization was enabled. This could lead to an incorrect query result. As a workaround, disable `do_not_merge_across_partitions_select_final` and `split_parts_ranges_into_intersecting_and_non_intersecting_final` before this fix is merged. [#65505](https://github.com/ClickHouse/ClickHouse/pull/65505) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#65788](https://github.com/ClickHouse/ClickHouse/issues/65788): Fixed bug in MergeJoin. Column in sparse serialisation might be treated as a column of its nested type though the required conversion wasn't performed. [#65632](https://github.com/ClickHouse/ClickHouse/pull/65632) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Backported in [#65812](https://github.com/ClickHouse/ClickHouse/issues/65812): Fix invalid exceptions in function `parseDateTime` with `%F` and `%D` placeholders. [#65768](https://github.com/ClickHouse/ClickHouse/pull/65768) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#65828](https://github.com/ClickHouse/ClickHouse/issues/65828): Fix a bug in short circuit logic when old analyzer and dictGetOrDefault is used. [#65802](https://github.com/ClickHouse/ClickHouse/pull/65802) ([jsc0218](https://github.com/jsc0218)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#65412](https://github.com/ClickHouse/ClickHouse/issues/65412): Re-enable OpenSSL session caching. [#65111](https://github.com/ClickHouse/ClickHouse/pull/65111) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Backported in [#65905](https://github.com/ClickHouse/ClickHouse/issues/65905): Fix bug with session closing in Keeper. [#65735](https://github.com/ClickHouse/ClickHouse/pull/65735) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
|
||||
@ -267,7 +267,7 @@ A pull request can be created even if the work is not completed yet. In this cas
|
||||
|
||||
Testing will commence as soon as ClickHouse employees label your PR with a tag “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour.
|
||||
|
||||
The system will prepare ClickHouse binary builds for your pull request individually. To retrieve these builds click the “Details” link next to “ClickHouse build check” entry in the list of checks. There you will find direct links to the built .deb packages of ClickHouse which you can deploy even on your production servers (if you have no fear).
|
||||
The system will prepare ClickHouse binary builds for your pull request individually. To retrieve these builds click the “Details” link next to “Builds” entry in the list of checks. There you will find direct links to the built .deb packages of ClickHouse which you can deploy even on your production servers (if you have no fear).
|
||||
|
||||
Most probably some of the builds will fail at first times. This is due to the fact that we check builds both with gcc as well as with clang, with almost all of existing warnings (always with the `-Werror` flag) enabled for clang. On that same page, you can find all of the build logs so that you do not have to build ClickHouse in all of the possible ways.
|
||||
|
||||
|
||||
@ -28,7 +28,7 @@ run, for example, the test `01428_hash_set_nan_key`, change to the repository
|
||||
folder and run the following command:
|
||||
|
||||
```
|
||||
PATH=$PATH:<path to clickhouse-client> tests/clickhouse-test 01428_hash_set_nan_key
|
||||
PATH=<path to clickhouse-client>:$PATH tests/clickhouse-test 01428_hash_set_nan_key
|
||||
```
|
||||
|
||||
Test results (`stderr` and `stdout`) are written to files `01428_hash_set_nan_key.[stderr|stdout]` which
|
||||
|
||||
@ -56,6 +56,15 @@ SELECT * FROM test_table;
|
||||
- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the size is unknown, the value is `NULL`.
|
||||
- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
|
||||
|
||||
## Authentication
|
||||
|
||||
Currently there are 3 ways to authenticate:
|
||||
- `Managed Identity` - Can be used by providing an `endpoint`, `connection_string` or `storage_account_url`.
|
||||
- `SAS Token` - Can be used by providing an `endpoint`, `connection_string` or `storage_account_url`. It is identified by presence of '?' in the url.
|
||||
- `Workload Identity` - Can be used by providing an `endpoint` or `storage_account_url`. If `use_workload_identity` parameter is set in config, ([workload identity](https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity#authenticate-azure-hosted-applications)) is used for authentication.
|
||||
|
||||
|
||||
|
||||
## See also
|
||||
|
||||
[Azure Blob Storage Table Function](/docs/en/sql-reference/table-functions/azureBlobStorage)
|
||||
|
||||
@ -58,7 +58,7 @@ Optional parameters:
|
||||
- `nats_max_reconnect` – Maximum amount of reconnection attempts per try to connect to NATS. Default: `5`.
|
||||
- `nats_reconnect_wait` – Amount of time in milliseconds to sleep between each reconnect attempt. Default: `5000`.
|
||||
- `nats_server_list` - Server list for connection. Can be specified to connect to NATS cluster.
|
||||
- `nats_skip_broken_messages` - NATS message parser tolerance to schema-incompatible messages per block. Default: `0`. If `nats_skip_broken_messages = N` then the engine skips *N* RabbitMQ messages that cannot be parsed (a message equals a row of data).
|
||||
- `nats_skip_broken_messages` - NATS message parser tolerance to schema-incompatible messages per block. Default: `0`. If `nats_skip_broken_messages = N` then the engine skips *N* NATS messages that cannot be parsed (a message equals a row of data).
|
||||
- `nats_max_block_size` - Number of row collected by poll(s) for flushing data from NATS. Default: [max_insert_block_size](../../../operations/settings/settings.md#max_insert_block_size).
|
||||
- `nats_flush_interval_ms` - Timeout for flushing data read from NATS. Default: [stream_flush_interval_ms](../../../operations/settings/settings.md#stream-flush-interval-ms).
|
||||
- `nats_username` - NATS username.
|
||||
@ -67,7 +67,7 @@ Optional parameters:
|
||||
- `nats_credential_file` - Path to a NATS credentials file.
|
||||
- `nats_startup_connect_tries` - Number of connect tries at startup. Default: `5`.
|
||||
- `nats_max_rows_per_message` — The maximum number of rows written in one NATS message for row-based formats. (default : `1`).
|
||||
- `nats_handle_error_mode` — How to handle errors for RabbitMQ engine. Possible values: default (the exception will be thrown if we fail to parse a message), stream (the exception message and raw message will be saved in virtual columns `_error` and `_raw_message`).
|
||||
- `nats_handle_error_mode` — How to handle errors for NATS engine. Possible values: default (the exception will be thrown if we fail to parse a message), stream (the exception message and raw message will be saved in virtual columns `_error` and `_raw_message`).
|
||||
|
||||
SSL connection:
|
||||
|
||||
|
||||
@ -28,6 +28,8 @@ CREATE TABLE s3_queue_engine_table (name String, value UInt32)
|
||||
[s3queue_cleanup_interval_max_ms = 30000,]
|
||||
```
|
||||
|
||||
Starting with `24.7` settings without `s3queue_` prefix are also supported.
|
||||
|
||||
**Engine parameters**
|
||||
|
||||
- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `**`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [below](#wildcards-in-path).
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 162 KiB |
394
docs/en/getting-started/example-datasets/stackoverflow.md
Normal file
394
docs/en/getting-started/example-datasets/stackoverflow.md
Normal file
@ -0,0 +1,394 @@
|
||||
---
|
||||
slug: /en/getting-started/example-datasets/stackoverflow
|
||||
sidebar_label: Stack Overflow
|
||||
sidebar_position: 1
|
||||
description: Analyzing Stack Overflow data with ClickHouse
|
||||
---
|
||||
|
||||
# Analyzing Stack Overflow data with ClickHouse
|
||||
|
||||
This dataset contains every `Post`, `User`, `Vote`, `Comment`, `Badge, `PostHistory`, and `PostLink` that has occurred on Stack Overflow.
|
||||
|
||||
Users can either download pre-prepared Parquet versions of the data, containing every post up to April 2024, or download the latest data in XML format and load this. Stack Overflow provide updates to this data periodically - historically every 3 months.
|
||||
|
||||
The following diagram shows the schema for the available tables assuming Parquet format.
|
||||
|
||||
|
||||
|
||||
A description of the schema of this data can be found [here](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede).
|
||||
|
||||
## Pre-prepared data
|
||||
|
||||
We provide a copy of this data in Parquet format, up to date as of April 2024. While small for ClickHouse with respect to the number of rows (60 million posts), this dataset contains significant volumes of text and large String columns.
|
||||
|
||||
```sql
|
||||
CREATE DATABASE stackoverflow
|
||||
```
|
||||
|
||||
The following timings are for a 96 GiB, 24 vCPU ClickHouse Cloud cluster located in `eu-west-2`. The dataset is located in `eu-west-3`.
|
||||
|
||||
### Posts
|
||||
|
||||
```sql
|
||||
CREATE TABLE stackoverflow.posts
|
||||
(
|
||||
`Id` Int32 CODEC(Delta(4), ZSTD(1)),
|
||||
`PostTypeId` Enum8('Question' = 1, 'Answer' = 2, 'Wiki' = 3, 'TagWikiExcerpt' = 4, 'TagWiki' = 5, 'ModeratorNomination' = 6, 'WikiPlaceholder' = 7, 'PrivilegeWiki' = 8),
|
||||
`AcceptedAnswerId` UInt32,
|
||||
`CreationDate` DateTime64(3, 'UTC'),
|
||||
`Score` Int32,
|
||||
`ViewCount` UInt32 CODEC(Delta(4), ZSTD(1)),
|
||||
`Body` String,
|
||||
`OwnerUserId` Int32,
|
||||
`OwnerDisplayName` String,
|
||||
`LastEditorUserId` Int32,
|
||||
`LastEditorDisplayName` String,
|
||||
`LastEditDate` DateTime64(3, 'UTC') CODEC(Delta(8), ZSTD(1)),
|
||||
`LastActivityDate` DateTime64(3, 'UTC'),
|
||||
`Title` String,
|
||||
`Tags` String,
|
||||
`AnswerCount` UInt16 CODEC(Delta(2), ZSTD(1)),
|
||||
`CommentCount` UInt8,
|
||||
`FavoriteCount` UInt8,
|
||||
`ContentLicense` LowCardinality(String),
|
||||
`ParentId` String,
|
||||
`CommunityOwnedDate` DateTime64(3, 'UTC'),
|
||||
`ClosedDate` DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYear(CreationDate)
|
||||
ORDER BY (PostTypeId, toDate(CreationDate), CreationDate)
|
||||
|
||||
INSERT INTO stackoverflow.posts SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/*.parquet')
|
||||
|
||||
0 rows in set. Elapsed: 265.466 sec. Processed 59.82 million rows, 38.07 GB (225.34 thousand rows/s., 143.42 MB/s.)
|
||||
```
|
||||
|
||||
Posts are also available by year e.g. [https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet](https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet)
|
||||
|
||||
|
||||
### Votes
|
||||
|
||||
```sql
|
||||
CREATE TABLE stackoverflow.votes
|
||||
(
|
||||
`Id` UInt32,
|
||||
`PostId` Int32,
|
||||
`VoteTypeId` UInt8,
|
||||
`CreationDate` DateTime64(3, 'UTC'),
|
||||
`UserId` Int32,
|
||||
`BountyAmount` UInt8
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (VoteTypeId, CreationDate, PostId, UserId)
|
||||
|
||||
INSERT INTO stackoverflow.votes SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/votes/*.parquet')
|
||||
|
||||
0 rows in set. Elapsed: 21.605 sec. Processed 238.98 million rows, 2.13 GB (11.06 million rows/s., 98.46 MB/s.)
|
||||
```
|
||||
|
||||
Votes are also available by year e.g. [https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet](https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/votes/2020.parquet)
|
||||
|
||||
|
||||
### Comments
|
||||
|
||||
```sql
|
||||
CREATE TABLE stackoverflow.comments
|
||||
(
|
||||
`Id` UInt32,
|
||||
`PostId` UInt32,
|
||||
`Score` UInt16,
|
||||
`Text` String,
|
||||
`CreationDate` DateTime64(3, 'UTC'),
|
||||
`UserId` Int32,
|
||||
`UserDisplayName` LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY CreationDate
|
||||
|
||||
INSERT INTO stackoverflow.comments SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/comments/*.parquet')
|
||||
|
||||
0 rows in set. Elapsed: 56.593 sec. Processed 90.38 million rows, 11.14 GB (1.60 million rows/s., 196.78 MB/s.)
|
||||
```
|
||||
|
||||
Comments are also available by year e.g. [https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet](https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/comments/2020.parquet)
|
||||
|
||||
### Users
|
||||
|
||||
```sql
|
||||
CREATE TABLE stackoverflow.users
|
||||
(
|
||||
`Id` Int32,
|
||||
`Reputation` LowCardinality(String),
|
||||
`CreationDate` DateTime64(3, 'UTC') CODEC(Delta(8), ZSTD(1)),
|
||||
`DisplayName` String,
|
||||
`LastAccessDate` DateTime64(3, 'UTC'),
|
||||
`AboutMe` String,
|
||||
`Views` UInt32,
|
||||
`UpVotes` UInt32,
|
||||
`DownVotes` UInt32,
|
||||
`WebsiteUrl` String,
|
||||
`Location` LowCardinality(String),
|
||||
`AccountId` Int32
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (Id, CreationDate)
|
||||
|
||||
INSERT INTO stackoverflow.users SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/users.parquet')
|
||||
|
||||
0 rows in set. Elapsed: 10.988 sec. Processed 22.48 million rows, 1.36 GB (2.05 million rows/s., 124.10 MB/s.)
|
||||
```
|
||||
|
||||
### Badges
|
||||
|
||||
```sql
|
||||
CREATE TABLE stackoverflow.badges
|
||||
(
|
||||
`Id` UInt32,
|
||||
`UserId` Int32,
|
||||
`Name` LowCardinality(String),
|
||||
`Date` DateTime64(3, 'UTC'),
|
||||
`Class` Enum8('Gold' = 1, 'Silver' = 2, 'Bronze' = 3),
|
||||
`TagBased` Bool
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY UserId
|
||||
|
||||
INSERT INTO stackoverflow.badges SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/badges.parquet')
|
||||
|
||||
0 rows in set. Elapsed: 6.635 sec. Processed 51.29 million rows, 797.05 MB (7.73 million rows/s., 120.13 MB/s.)
|
||||
```
|
||||
|
||||
### `PostLinks`
|
||||
|
||||
```sql
|
||||
CREATE TABLE stackoverflow.postlinks
|
||||
(
|
||||
`Id` UInt64,
|
||||
`CreationDate` DateTime64(3, 'UTC'),
|
||||
`PostId` Int32,
|
||||
`RelatedPostId` Int32,
|
||||
`LinkTypeId` Enum8('Linked' = 1, 'Duplicate' = 3)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (PostId, RelatedPostId)
|
||||
|
||||
INSERT INTO stackoverflow.postlinks SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/postlinks.parquet')
|
||||
|
||||
0 rows in set. Elapsed: 1.534 sec. Processed 6.55 million rows, 129.70 MB (4.27 million rows/s., 84.57 MB/s.)
|
||||
```
|
||||
|
||||
### `PostHistory`
|
||||
|
||||
```sql
|
||||
CREATE TABLE stackoverflow.posthistory
|
||||
(
|
||||
`Id` UInt64,
|
||||
`PostHistoryTypeId` UInt8,
|
||||
`PostId` Int32,
|
||||
`RevisionGUID` String,
|
||||
`CreationDate` DateTime64(3, 'UTC'),
|
||||
`UserId` Int32,
|
||||
`Text` String,
|
||||
`ContentLicense` LowCardinality(String),
|
||||
`Comment` String,
|
||||
`UserDisplayName` String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (CreationDate, PostId)
|
||||
|
||||
INSERT INTO stackoverflow.posthistory SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posthistory/*.parquet')
|
||||
|
||||
0 rows in set. Elapsed: 422.795 sec. Processed 160.79 million rows, 67.08 GB (380.30 thousand rows/s., 158.67 MB/s.)
|
||||
```
|
||||
|
||||
## Original dataset
|
||||
|
||||
The original dataset is available in compressed (7zip) XML format at [https://archive.org/download/stackexchange](https://archive.org/download/stackexchange) - files with prefix `stackoverflow.com*`.
|
||||
|
||||
### Download
|
||||
|
||||
```bash
|
||||
wget https://archive.org/download/stackexchange/stackoverflow.com-Badges.7z
|
||||
wget https://archive.org/download/stackexchange/stackoverflow.com-Comments.7z
|
||||
wget https://archive.org/download/stackexchange/stackoverflow.com-PostHistory.7z
|
||||
wget https://archive.org/download/stackexchange/stackoverflow.com-PostLinks.7z
|
||||
wget https://archive.org/download/stackexchange/stackoverflow.com-Posts.7z
|
||||
wget https://archive.org/download/stackexchange/stackoverflow.com-Users.7z
|
||||
wget https://archive.org/download/stackexchange/stackoverflow.com-Votes.7z
|
||||
```
|
||||
|
||||
These files are up to 35GB and can take around 30 mins to download depending on internet connection - the download server throttles at around 20MB/sec.
|
||||
|
||||
### Convert to JSON
|
||||
|
||||
At the time of writing, ClickHouse does not have native support for XML as an input format. To load the data into ClickHouse we first convert to NDJSON.
|
||||
|
||||
To convert XML to JSON we recommend the [`xq`](https://github.com/kislyuk/yq) linux tool, a simple `jq` wrapper for XML documents.
|
||||
|
||||
Install xq and jq:
|
||||
|
||||
```bash
|
||||
sudo apt install jq
|
||||
pip install yq
|
||||
```
|
||||
|
||||
The following steps apply to any of the above files. We use the `stackoverflow.com-Posts.7z` file as an example. Modify as required.
|
||||
|
||||
Extract the file using [p7zip](https://p7zip.sourceforge.net/). This will produce a single xml file - in this case `Posts.xml`.
|
||||
|
||||
> Files are compressed approximately 4.5x. At 22GB compressed, the posts file requires around 97G uncompressed.
|
||||
|
||||
```bash
|
||||
p7zip -d stackoverflow.com-Posts.7z
|
||||
```
|
||||
|
||||
The following splits the xml file into files, each containing 10000 rows.
|
||||
|
||||
```bash
|
||||
mkdir posts
|
||||
cd posts
|
||||
# the following splits the input xml file into sub files of 10000 rows
|
||||
tail +3 ../Posts.xml | head -n -1 | split -l 10000 --filter='{ printf "<rows>\n"; cat - ; printf "</rows>\n"; } > $FILE' -
|
||||
```
|
||||
|
||||
After running the above users will have a set of files, each with 10000 lines. This ensures the memory overhead of the next command is not excessive (xml to JSON conversion is done in memory).
|
||||
|
||||
```bash
|
||||
find . -maxdepth 1 -type f -exec xq -c '.rows.row[]' {} \; | sed -e 's:"@:":g' > posts_v2.json
|
||||
```
|
||||
|
||||
The above command will produce a single `posts.json` file.
|
||||
|
||||
Load into ClickHouse with the following command. Note the schema is specified for the `posts.json` file. This will need to be adjusted per data type to align with the target table.
|
||||
|
||||
```bash
|
||||
clickhouse local --query "SELECT * FROM file('posts.json', JSONEachRow, 'Id Int32, PostTypeId UInt8, AcceptedAnswerId UInt32, CreationDate DateTime64(3, \'UTC\'), Score Int32, ViewCount UInt32, Body String, OwnerUserId Int32, OwnerDisplayName String, LastEditorUserId Int32, LastEditorDisplayName String, LastEditDate DateTime64(3, \'UTC\'), LastActivityDate DateTime64(3, \'UTC\'), Title String, Tags String, AnswerCount UInt16, CommentCount UInt8, FavoriteCount UInt8, ContentLicense String, ParentId String, CommunityOwnedDate DateTime64(3, \'UTC\'), ClosedDate DateTime64(3, \'UTC\')') FORMAT Native" | clickhouse client --host <host> --secure --password <password> --query "INSERT INTO stackoverflow.posts_v2 FORMAT Native"
|
||||
```
|
||||
|
||||
## Example queries
|
||||
|
||||
A few simple questions to you get started.
|
||||
|
||||
### Most popular tags on Stack Overflow
|
||||
|
||||
```sql
|
||||
|
||||
SELECT
|
||||
arrayJoin(arrayFilter(t -> (t != ''), splitByChar('|', Tags))) AS Tags,
|
||||
count() AS c
|
||||
FROM stackoverflow.posts
|
||||
GROUP BY Tags
|
||||
ORDER BY c DESC
|
||||
LIMIT 10
|
||||
|
||||
┌─Tags───────┬───────c─┐
|
||||
│ javascript │ 2527130 │
|
||||
│ python │ 2189638 │
|
||||
│ java │ 1916156 │
|
||||
│ c# │ 1614236 │
|
||||
│ php │ 1463901 │
|
||||
│ android │ 1416442 │
|
||||
│ html │ 1186567 │
|
||||
│ jquery │ 1034621 │
|
||||
│ c++ │ 806202 │
|
||||
│ css │ 803755 │
|
||||
└────────────┴─────────┘
|
||||
|
||||
10 rows in set. Elapsed: 1.013 sec. Processed 59.82 million rows, 1.21 GB (59.07 million rows/s., 1.19 GB/s.)
|
||||
Peak memory usage: 224.03 MiB.
|
||||
```
|
||||
|
||||
### User with the most answers (active accounts)
|
||||
|
||||
Account requires a `UserId`.
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
any(OwnerUserId) UserId,
|
||||
OwnerDisplayName,
|
||||
count() AS c
|
||||
FROM stackoverflow.posts WHERE OwnerDisplayName != '' AND PostTypeId='Answer' AND OwnerUserId != 0
|
||||
GROUP BY OwnerDisplayName
|
||||
ORDER BY c DESC
|
||||
LIMIT 5
|
||||
|
||||
┌─UserId─┬─OwnerDisplayName─┬────c─┐
|
||||
│ 22656 │ Jon Skeet │ 2727 │
|
||||
│ 23354 │ Marc Gravell │ 2150 │
|
||||
│ 12950 │ tvanfosson │ 1530 │
|
||||
│ 3043 │ Joel Coehoorn │ 1438 │
|
||||
│ 10661 │ S.Lott │ 1087 │
|
||||
└────────┴──────────────────┴──────┘
|
||||
|
||||
5 rows in set. Elapsed: 0.154 sec. Processed 35.83 million rows, 193.39 MB (232.33 million rows/s., 1.25 GB/s.)
|
||||
Peak memory usage: 206.45 MiB.
|
||||
```
|
||||
|
||||
### ClickHouse related posts with the most views
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
Id,
|
||||
Title,
|
||||
ViewCount,
|
||||
AnswerCount
|
||||
FROM stackoverflow.posts
|
||||
WHERE Title ILIKE '%ClickHouse%'
|
||||
ORDER BY ViewCount DESC
|
||||
LIMIT 10
|
||||
|
||||
┌───────Id─┬─Title────────────────────────────────────────────────────────────────────────────┬─ViewCount─┬─AnswerCount─┐
|
||||
│ 52355143 │ Is it possible to delete old records from clickhouse table? │ 41462 │ 3 │
|
||||
│ 37954203 │ Clickhouse Data Import │ 38735 │ 3 │
|
||||
│ 37901642 │ Updating data in Clickhouse │ 36236 │ 6 │
|
||||
│ 58422110 │ Pandas: How to insert dataframe into Clickhouse │ 29731 │ 4 │
|
||||
│ 63621318 │ DBeaver - Clickhouse - SQL Error [159] .. Read timed out │ 27350 │ 1 │
|
||||
│ 47591813 │ How to filter clickhouse table by array column contents? │ 27078 │ 2 │
|
||||
│ 58728436 │ How to search the string in query with case insensitive on Clickhouse database? │ 26567 │ 3 │
|
||||
│ 65316905 │ Clickhouse: DB::Exception: Memory limit (for query) exceeded │ 24899 │ 2 │
|
||||
│ 49944865 │ How to add a column in clickhouse │ 24424 │ 1 │
|
||||
│ 59712399 │ How to cast date Strings to DateTime format with extended parsing in ClickHouse? │ 22620 │ 1 │
|
||||
└──────────┴──────────────────────────────────────────────────────────────────────────────────┴───────────┴─────────────┘
|
||||
|
||||
10 rows in set. Elapsed: 0.472 sec. Processed 59.82 million rows, 1.91 GB (126.63 million rows/s., 4.03 GB/s.)
|
||||
Peak memory usage: 240.01 MiB.
|
||||
```
|
||||
|
||||
### Most controversial posts
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
Id,
|
||||
Title,
|
||||
UpVotes,
|
||||
DownVotes,
|
||||
abs(UpVotes - DownVotes) AS Controversial_ratio
|
||||
FROM stackoverflow.posts
|
||||
INNER JOIN
|
||||
(
|
||||
SELECT
|
||||
PostId,
|
||||
countIf(VoteTypeId = 2) AS UpVotes,
|
||||
countIf(VoteTypeId = 3) AS DownVotes
|
||||
FROM stackoverflow.votes
|
||||
GROUP BY PostId
|
||||
HAVING (UpVotes > 10) AND (DownVotes > 10)
|
||||
) AS votes ON posts.Id = votes.PostId
|
||||
WHERE Title != ''
|
||||
ORDER BY Controversial_ratio ASC
|
||||
LIMIT 3
|
||||
|
||||
┌───────Id─┬─Title─────────────────────────────────────────────┬─UpVotes─┬─DownVotes─┬─Controversial_ratio─┐
|
||||
│ 583177 │ VB.NET Infinite For Loop │ 12 │ 12 │ 0 │
|
||||
│ 9756797 │ Read console input as enumerable - one statement? │ 16 │ 16 │ 0 │
|
||||
│ 13329132 │ What's the point of ARGV in Ruby? │ 22 │ 22 │ 0 │
|
||||
└──────────┴───────────────────────────────────────────────────┴─────────┴───────────┴─────────────────────┘
|
||||
|
||||
3 rows in set. Elapsed: 4.779 sec. Processed 298.80 million rows, 3.16 GB (62.52 million rows/s., 661.05 MB/s.)
|
||||
Peak memory usage: 6.05 GiB.
|
||||
```
|
||||
|
||||
## Attribution
|
||||
|
||||
We thank Stack Overflow for providing this data under the `cc-by-sa 4.0` license, acknowledging their efforts and the original source of the data at [https://archive.org/details/stackexchange](https://archive.org/details/stackexchange).
|
||||
@ -314,7 +314,7 @@ For example, to download a aarch64 binary for ClickHouse v23.4, follow these ste
|
||||
- Find the GitHub pull request for release v23.4: [Release pull request for branch 23.4](https://github.com/ClickHouse/ClickHouse/pull/49238)
|
||||
- Click "Commits", then click a commit similar to "Update autogenerated version to 23.4.2.1 and contributors" for the particular version you like to install.
|
||||
- Click the green check / yellow dot / red cross to open the list of CI checks.
|
||||
- Click "Details" next to "ClickHouse Build Check" in the list, it will open a page similar to [this page](https://s3.amazonaws.com/clickhouse-test-reports/46793/b460eb70bf29b19eadd19a1f959b15d186705394/clickhouse_build_check/report.html)
|
||||
- Click "Details" next to "Builds" in the list, it will open a page similar to [this page](https://s3.amazonaws.com/clickhouse-test-reports/46793/b460eb70bf29b19eadd19a1f959b15d186705394/clickhouse_build_check/report.html)
|
||||
- Find the rows with compiler = "clang-*-aarch64" - there are multiple rows.
|
||||
- Download the artifacts for these builds.
|
||||
|
||||
|
||||
@ -193,6 +193,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
|
||||
- `--hardware-utilization` — Print hardware utilization information in progress bar.
|
||||
- `--print-profile-events` – Print `ProfileEvents` packets.
|
||||
- `--profile-events-delay-ms` – Delay between printing `ProfileEvents` packets (-1 - print only totals, 0 - print every single packet).
|
||||
- `--jwt` – If specified, enables authorization via JSON Web Token. Server JWT authorization is available only in ClickHouse Cloud.
|
||||
|
||||
Instead of `--host`, `--port`, `--user` and `--password` options, ClickHouse client also supports connection strings (see next section).
|
||||
|
||||
|
||||
@ -2169,6 +2169,7 @@ To exchange data with Hadoop, you can use [HDFS table engine](/docs/en/engines/t
|
||||
- [output_format_parquet_compression_method](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_compression_method) - compression method used in output Parquet format. Default value - `lz4`.
|
||||
- [input_format_parquet_max_block_size](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_max_block_size) - Max block row size for parquet reader. Default value - `65409`.
|
||||
- [input_format_parquet_prefer_block_bytes](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_prefer_block_bytes) - Average block bytes output by parquet reader. Default value - `16744704`.
|
||||
- [output_format_parquet_write_page_index](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_max_block_size) - Add a possibility to write page index into parquet files. Need to disable `output_format_parquet_use_custom_encoder` at present. Default value - `true`.
|
||||
|
||||
## ParquetMetadata {data-format-parquet-metadata}
|
||||
|
||||
|
||||
@ -5,6 +5,10 @@ sidebar_label: "Named collections"
|
||||
title: "Named collections"
|
||||
---
|
||||
|
||||
import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
|
||||
|
||||
<CloudNotSupportedBadge />
|
||||
|
||||
Named collections provide a way to store collections of key-value pairs to be
|
||||
used to configure integrations with external sources. You can use named collections with
|
||||
dictionaries, tables, table functions, and object storage.
|
||||
|
||||
@ -498,6 +498,8 @@ Default: 0.9
|
||||
Interval in seconds during which the server's maximum allowed memory consumption is adjusted by the corresponding threshold in cgroups. (see
|
||||
settings `cgroup_memory_watcher_hard_limit_ratio` and `cgroup_memory_watcher_soft_limit_ratio`).
|
||||
|
||||
To disable the cgroup observer, set this value to `0`.
|
||||
|
||||
Type: UInt64
|
||||
|
||||
Default: 15
|
||||
@ -1463,6 +1465,9 @@ Keys:
|
||||
- `size` – Size of the file. Applies to `log` and `errorlog`. Once the file reaches `size`, ClickHouse archives and renames it, and creates a new log file in its place.
|
||||
- `count` – The number of archived log files that ClickHouse stores.
|
||||
- `console` – Send `log` and `errorlog` to the console instead of file. To enable, set to `1` or `true`.
|
||||
- `console_log_level` – Logging level for console. Default to `level`.
|
||||
- `use_syslog` - Log to syslog as well.
|
||||
- `syslog_level` - Logging level for logging to syslog.
|
||||
- `stream_compress` – Compress `log` and `errorlog` with `lz4` stream compression. To enable, set to `1` or `true`.
|
||||
- `formatting` – Specify log format to be printed in console log (currently only `json` supported).
|
||||
|
||||
|
||||
@ -1428,6 +1428,13 @@ Average block bytes output by parquet reader. Lowering the configuration in the
|
||||
|
||||
Default value: `65409 * 256 = 16744704`
|
||||
|
||||
### output_format_parquet_write_page_index {#input_format_parquet_max_block_size}
|
||||
|
||||
Could add page index into parquet files. To enable this, need set `output_format_parquet_use_custom_encoder`=`false` and
|
||||
`output_format_parquet_write_page_index`=`true`.
|
||||
|
||||
Enable by default.
|
||||
|
||||
## Hive format settings {#hive-format-settings}
|
||||
|
||||
### input_format_hive_text_fields_delimiter {#input_format_hive_text_fields_delimiter}
|
||||
|
||||
@ -5418,11 +5418,14 @@ When set to `false` than all attempts are made with identical timeouts.
|
||||
|
||||
Default value: `true`.
|
||||
|
||||
## uniform_snowflake_conversion_functions {#uniform_snowflake_conversion_functions}
|
||||
## allow_deprecated_snowflake_conversion_functions {#allow_deprecated_snowflake_conversion_functions}
|
||||
|
||||
If set to `true`, then functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` are enabled, and functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake` are disabled (and vice versa if set to `false`).
|
||||
Functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake` are deprecated and disabled by default.
|
||||
Please use functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` instead.
|
||||
|
||||
Default value: `true`
|
||||
To re-enable the deprecated functions (e.g., during a transition period), please set this setting to `true`.
|
||||
|
||||
Default value: `false`
|
||||
|
||||
## allow_experimental_variant_type {#allow_experimental_variant_type}
|
||||
|
||||
|
||||
@ -36,9 +36,24 @@ $ echo 0 | sudo tee /proc/sys/vm/overcommit_memory
|
||||
Use `perf top` to watch the time spent in the kernel for memory management.
|
||||
Permanent huge pages also do not need to be allocated.
|
||||
|
||||
:::warning
|
||||
If your system has less than 16 GB of RAM, you may experience various memory exceptions because default settings do not match this amount of memory. The recommended amount of RAM is 32 GB or more. You can use ClickHouse in a system with a small amount of RAM, even with 2 GB of RAM, but it requires additional tuning and can ingest at a low rate.
|
||||
:::
|
||||
### Using less than 16GB of RAM
|
||||
|
||||
The recommended amount of RAM is 32 GB or more.
|
||||
|
||||
If your system has less than 16 GB of RAM, you may experience various memory exceptions because default settings do not match this amount of memory. You can use ClickHouse in a system with a small amount of RAM (as low as 2 GB), but these setups require additional tuning and can only ingest at a low rate.
|
||||
|
||||
When using ClickHouse with less than 16GB of RAM, we recommend the following:
|
||||
|
||||
- Lower the size of the mark cache in the `config.xml`. It can be set as low as 500 MB, but it cannot be set to zero.
|
||||
- Lower the number of query processing threads down to `1`.
|
||||
- Lower the `max_block_size` to `8192`. Values as low as `1024` can still be practical.
|
||||
- Lower `max_download_threads` to `1`.
|
||||
- Set `input_format_parallel_parsing` and `output_format_parallel_formatting` to `0`.
|
||||
|
||||
Additional notes:
|
||||
- To flush the memory cached by the memory allocator, you can run the `SYSTEM JEMALLOC PURGE`
|
||||
command.
|
||||
- We do not recommend using S3 or Kafka integrations on low-memory machines because they require significant memory for buffers.
|
||||
|
||||
## Storage Subsystem {#storage-subsystem}
|
||||
|
||||
|
||||
@ -236,10 +236,10 @@ Read 2 rows, 32.00 B in 0.000 sec., 5182 rows/sec., 80.97 KiB/sec.
|
||||
Previous example is the same as:
|
||||
|
||||
``` bash
|
||||
$ echo -e "1,2\n3,4" | clickhouse-local --query "
|
||||
$ echo -e "1,2\n3,4" | clickhouse-local -n --query "
|
||||
CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin);
|
||||
SELECT a, b FROM table;
|
||||
DROP TABLE table"
|
||||
DROP TABLE table;"
|
||||
Read 2 rows, 32.00 B in 0.000 sec., 4987 rows/sec., 77.93 KiB/sec.
|
||||
1 2
|
||||
3 4
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/analysis_of_variance
|
||||
sidebar_position: 6
|
||||
sidebar_position: 101
|
||||
---
|
||||
|
||||
# analysisOfVariance
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/any
|
||||
sidebar_position: 6
|
||||
sidebar_position: 102
|
||||
---
|
||||
|
||||
# any
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/anyheavy
|
||||
sidebar_position: 103
|
||||
sidebar_position: 104
|
||||
---
|
||||
|
||||
# anyHeavy
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/anylast
|
||||
sidebar_position: 104
|
||||
sidebar_position: 105
|
||||
---
|
||||
|
||||
# anyLast
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/anylast_respect_nulls
|
||||
sidebar_position: 104
|
||||
sidebar_position: 106
|
||||
---
|
||||
|
||||
# anyLast_respect_nulls
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/approxtopk
|
||||
sidebar_position: 212
|
||||
sidebar_position: 107
|
||||
---
|
||||
|
||||
# approx_top_k
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/approxtopsum
|
||||
sidebar_position: 212
|
||||
sidebar_position: 108
|
||||
---
|
||||
|
||||
# approx_top_sum
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/argmax
|
||||
sidebar_position: 106
|
||||
sidebar_position: 109
|
||||
---
|
||||
|
||||
# argMax
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/argmin
|
||||
sidebar_position: 105
|
||||
sidebar_position: 110
|
||||
---
|
||||
|
||||
# argMin
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/array_concat_agg
|
||||
sidebar_position: 110
|
||||
sidebar_position: 111
|
||||
---
|
||||
|
||||
# array_concat_agg
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/avg
|
||||
sidebar_position: 5
|
||||
sidebar_position: 112
|
||||
---
|
||||
|
||||
# avg
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/avgweighted
|
||||
sidebar_position: 107
|
||||
sidebar_position: 113
|
||||
---
|
||||
|
||||
# avgWeighted
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/boundingRatio
|
||||
sidebar_position: 2
|
||||
sidebar_position: 114
|
||||
title: boundingRatio
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/categoricalinformationvalue
|
||||
sidebar_position: 250
|
||||
sidebar_position: 115
|
||||
title: categoricalInformationValue
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/contingency
|
||||
sidebar_position: 350
|
||||
sidebar_position: 116
|
||||
---
|
||||
|
||||
# contingency
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/corr
|
||||
sidebar_position: 107
|
||||
sidebar_position: 117
|
||||
---
|
||||
|
||||
# corr
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/corrmatrix
|
||||
sidebar_position: 108
|
||||
sidebar_position: 118
|
||||
---
|
||||
|
||||
# corrMatrix
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/corrstable
|
||||
sidebar_position: 107
|
||||
sidebar_position: 119
|
||||
---
|
||||
|
||||
# corrStable
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/count
|
||||
sidebar_position: 1
|
||||
sidebar_position: 120
|
||||
---
|
||||
|
||||
# count
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/covarpop
|
||||
sidebar_position: 37
|
||||
sidebar_position: 121
|
||||
---
|
||||
|
||||
# covarPop
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/covarpopmatrix
|
||||
sidebar_position: 36
|
||||
sidebar_position: 122
|
||||
---
|
||||
|
||||
# covarPopMatrix
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/covarpopstable
|
||||
sidebar_position: 36
|
||||
sidebar_position: 123
|
||||
---
|
||||
|
||||
# covarPopStable
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/covarsamp
|
||||
sidebar_position: 37
|
||||
sidebar_position: 124
|
||||
---
|
||||
|
||||
# covarSamp
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/covarsampmatrix
|
||||
sidebar_position: 38
|
||||
sidebar_position: 125
|
||||
---
|
||||
|
||||
# covarSampMatrix
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/covarsampstable
|
||||
sidebar_position: 37
|
||||
sidebar_position: 126
|
||||
---
|
||||
|
||||
# covarSampStable
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/cramersv
|
||||
sidebar_position: 351
|
||||
sidebar_position: 127
|
||||
---
|
||||
|
||||
# cramersV
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/cramersvbiascorrected
|
||||
sidebar_position: 352
|
||||
sidebar_position: 128
|
||||
---
|
||||
|
||||
# cramersVBiasCorrected
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/deltasum
|
||||
sidebar_position: 141
|
||||
sidebar_position: 129
|
||||
---
|
||||
|
||||
# deltaSum
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/deltasumtimestamp
|
||||
sidebar_position: 141
|
||||
sidebar_position: 130
|
||||
title: deltaSumTimestamp
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/entropy
|
||||
sidebar_position: 302
|
||||
sidebar_position: 131
|
||||
---
|
||||
|
||||
# entropy
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/exponentialmovingaverage
|
||||
sidebar_position: 108
|
||||
sidebar_title: exponentialMovingAverage
|
||||
slug: /en/sql-reference/aggregate-functions/reference/exponentialMovingAverage
|
||||
sidebar_position: 132
|
||||
title: exponentialMovingAverage
|
||||
---
|
||||
|
||||
## exponentialMovingAverage
|
||||
@ -96,56 +96,56 @@ Result:
|
||||
|
||||
``` text
|
||||
┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────────────────────────────────────┐
|
||||
│ 1 │ 0 │ 0.067 │ ███▎ │
|
||||
│ 1 │ 0 │ 0.067 │ ███▎ │
|
||||
│ 0 │ 1 │ 0.062 │ ███ │
|
||||
│ 0 │ 2 │ 0.058 │ ██▊ │
|
||||
│ 0 │ 3 │ 0.054 │ ██▋ │
|
||||
│ 0 │ 2 │ 0.058 │ ██▊ │
|
||||
│ 0 │ 3 │ 0.054 │ ██▋ │
|
||||
│ 0 │ 4 │ 0.051 │ ██▌ │
|
||||
│ 0 │ 5 │ 0.047 │ ██▎ │
|
||||
│ 0 │ 6 │ 0.044 │ ██▏ │
|
||||
│ 0 │ 5 │ 0.047 │ ██▎ │
|
||||
│ 0 │ 6 │ 0.044 │ ██▏ │
|
||||
│ 0 │ 7 │ 0.041 │ ██ │
|
||||
│ 0 │ 8 │ 0.038 │ █▊ │
|
||||
│ 0 │ 9 │ 0.036 │ █▋ │
|
||||
│ 0 │ 10 │ 0.033 │ █▋ │
|
||||
│ 0 │ 8 │ 0.038 │ █▊ │
|
||||
│ 0 │ 9 │ 0.036 │ █▋ │
|
||||
│ 0 │ 10 │ 0.033 │ █▋ │
|
||||
│ 0 │ 11 │ 0.031 │ █▌ │
|
||||
│ 0 │ 12 │ 0.029 │ █▍ │
|
||||
│ 0 │ 13 │ 0.027 │ █▎ │
|
||||
│ 0 │ 14 │ 0.025 │ █▎ │
|
||||
│ 0 │ 15 │ 0.024 │ █▏ │
|
||||
│ 0 │ 12 │ 0.029 │ █▍ │
|
||||
│ 0 │ 13 │ 0.027 │ █▎ │
|
||||
│ 0 │ 14 │ 0.025 │ █▎ │
|
||||
│ 0 │ 15 │ 0.024 │ █▏ │
|
||||
│ 0 │ 16 │ 0.022 │ █ │
|
||||
│ 0 │ 17 │ 0.021 │ █ │
|
||||
│ 0 │ 18 │ 0.019 │ ▊ │
|
||||
│ 0 │ 19 │ 0.018 │ ▊ │
|
||||
│ 0 │ 20 │ 0.017 │ ▋ │
|
||||
│ 0 │ 21 │ 0.016 │ ▋ │
|
||||
│ 0 │ 22 │ 0.015 │ ▋ │
|
||||
│ 0 │ 23 │ 0.014 │ ▋ │
|
||||
│ 0 │ 24 │ 0.013 │ ▋ │
|
||||
│ 1 │ 25 │ 0.079 │ ███▊ │
|
||||
│ 0 │ 18 │ 0.019 │ ▊ │
|
||||
│ 0 │ 19 │ 0.018 │ ▊ │
|
||||
│ 0 │ 20 │ 0.017 │ ▋ │
|
||||
│ 0 │ 21 │ 0.016 │ ▋ │
|
||||
│ 0 │ 22 │ 0.015 │ ▋ │
|
||||
│ 0 │ 23 │ 0.014 │ ▋ │
|
||||
│ 0 │ 24 │ 0.013 │ ▋ │
|
||||
│ 1 │ 25 │ 0.079 │ ███▊ │
|
||||
│ 1 │ 26 │ 0.14 │ ███████ │
|
||||
│ 1 │ 27 │ 0.198 │ █████████▊ │
|
||||
│ 1 │ 27 │ 0.198 │ █████████▊ │
|
||||
│ 1 │ 28 │ 0.252 │ ████████████▌ │
|
||||
│ 1 │ 29 │ 0.302 │ ███████████████ │
|
||||
│ 1 │ 30 │ 0.349 │ █████████████████▍ │
|
||||
│ 1 │ 30 │ 0.349 │ █████████████████▍ │
|
||||
│ 1 │ 31 │ 0.392 │ ███████████████████▌ │
|
||||
│ 1 │ 32 │ 0.433 │ █████████████████████▋ │
|
||||
│ 1 │ 32 │ 0.433 │ █████████████████████▋ │
|
||||
│ 1 │ 33 │ 0.471 │ ███████████████████████▌ │
|
||||
│ 1 │ 34 │ 0.506 │ █████████████████████████▎ │
|
||||
│ 1 │ 35 │ 0.539 │ ██████████████████████████▊ │
|
||||
│ 1 │ 34 │ 0.506 │ █████████████████████████▎ │
|
||||
│ 1 │ 35 │ 0.539 │ ██████████████████████████▊ │
|
||||
│ 1 │ 36 │ 0.57 │ ████████████████████████████▌ │
|
||||
│ 1 │ 37 │ 0.599 │ █████████████████████████████▊ │
|
||||
│ 1 │ 38 │ 0.626 │ ███████████████████████████████▎ │
|
||||
│ 1 │ 37 │ 0.599 │ █████████████████████████████▊ │
|
||||
│ 1 │ 38 │ 0.626 │ ███████████████████████████████▎ │
|
||||
│ 1 │ 39 │ 0.651 │ ████████████████████████████████▌ │
|
||||
│ 1 │ 40 │ 0.674 │ █████████████████████████████████▋ │
|
||||
│ 1 │ 41 │ 0.696 │ ██████████████████████████████████▋ │
|
||||
│ 1 │ 42 │ 0.716 │ ███████████████████████████████████▋ │
|
||||
│ 1 │ 43 │ 0.735 │ ████████████████████████████████████▋ │
|
||||
│ 1 │ 44 │ 0.753 │ █████████████████████████████████████▋ │
|
||||
│ 1 │ 45 │ 0.77 │ ██████████████████████████████████████▍ │
|
||||
│ 1 │ 46 │ 0.785 │ ███████████████████████████████████████▎ │
|
||||
│ 1 │ 47 │ 0.8 │ ███████████████████████████████████████▊ │
|
||||
│ 1 │ 48 │ 0.813 │ ████████████████████████████████████████▋ │
|
||||
│ 1 │ 49 │ 0.825 │ █████████████████████████████████████████▎│
|
||||
│ 1 │ 40 │ 0.674 │ █████████████████████████████████▋ │
|
||||
│ 1 │ 41 │ 0.696 │ ██████████████████████████████████▋ │
|
||||
│ 1 │ 42 │ 0.716 │ ███████████████████████████████████▋ │
|
||||
│ 1 │ 43 │ 0.735 │ ████████████████████████████████████▋ │
|
||||
│ 1 │ 44 │ 0.753 │ █████████████████████████████████████▋ │
|
||||
│ 1 │ 45 │ 0.77 │ ██████████████████████████████████████▍ │
|
||||
│ 1 │ 46 │ 0.785 │ ███████████████████████████████████████▎ │
|
||||
│ 1 │ 47 │ 0.8 │ ███████████████████████████████████████▊ │
|
||||
│ 1 │ 48 │ 0.813 │ ████████████████████████████████████████▋ │
|
||||
│ 1 │ 49 │ 0.825 │ █████████████████████████████████████████▎ │
|
||||
└───────┴──────┴──────────────────────┴────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
@ -0,0 +1,105 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/exponentialTimeDecayedAvg
|
||||
sidebar_position: 133
|
||||
title: exponentialTimeDecayedAvg
|
||||
---
|
||||
|
||||
## exponentialTimeDecayedAvg
|
||||
|
||||
Returns the exponentially smoothed weighted moving average of values of a time series at point `t` in time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
exponentialTimeDecayedAvg(x)(v, t)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `v` — Value. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
- `t` — Time. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md), [DateTime](../../data-types/datetime.md), [DateTime64](../../data-types/datetime64.md).
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x` — Half-life period. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned values**
|
||||
|
||||
- Returns an exponentially smoothed weighted moving average at index `t` in time. [Float64](../../data-types/float.md).
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
value,
|
||||
time,
|
||||
round(exp_smooth, 3),
|
||||
bar(exp_smooth, 0, 5, 50) AS bar
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
(number = 0) OR (number >= 25) AS value,
|
||||
number AS time,
|
||||
exponentialTimeDecayedAvg(10)(value, time) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
|
||||
FROM numbers(50)
|
||||
);
|
||||
```
|
||||
|
||||
Response:
|
||||
|
||||
```sql
|
||||
┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────┐
|
||||
1. │ 1 │ 0 │ 1 │ ██████████ │
|
||||
2. │ 0 │ 1 │ 0.475 │ ████▊ │
|
||||
3. │ 0 │ 2 │ 0.301 │ ███ │
|
||||
4. │ 0 │ 3 │ 0.214 │ ██▏ │
|
||||
5. │ 0 │ 4 │ 0.162 │ █▌ │
|
||||
6. │ 0 │ 5 │ 0.128 │ █▎ │
|
||||
7. │ 0 │ 6 │ 0.104 │ █ │
|
||||
8. │ 0 │ 7 │ 0.086 │ ▊ │
|
||||
9. │ 0 │ 8 │ 0.072 │ ▋ │
|
||||
0. │ 0 │ 9 │ 0.061 │ ▌ │
|
||||
1. │ 0 │ 10 │ 0.052 │ ▌ │
|
||||
2. │ 0 │ 11 │ 0.045 │ ▍ │
|
||||
3. │ 0 │ 12 │ 0.039 │ ▍ │
|
||||
4. │ 0 │ 13 │ 0.034 │ ▎ │
|
||||
5. │ 0 │ 14 │ 0.03 │ ▎ │
|
||||
6. │ 0 │ 15 │ 0.027 │ ▎ │
|
||||
7. │ 0 │ 16 │ 0.024 │ ▏ │
|
||||
8. │ 0 │ 17 │ 0.021 │ ▏ │
|
||||
9. │ 0 │ 18 │ 0.018 │ ▏ │
|
||||
0. │ 0 │ 19 │ 0.016 │ ▏ │
|
||||
1. │ 0 │ 20 │ 0.015 │ ▏ │
|
||||
2. │ 0 │ 21 │ 0.013 │ ▏ │
|
||||
3. │ 0 │ 22 │ 0.012 │ │
|
||||
4. │ 0 │ 23 │ 0.01 │ │
|
||||
5. │ 0 │ 24 │ 0.009 │ │
|
||||
6. │ 1 │ 25 │ 0.111 │ █ │
|
||||
7. │ 1 │ 26 │ 0.202 │ ██ │
|
||||
8. │ 1 │ 27 │ 0.283 │ ██▊ │
|
||||
9. │ 1 │ 28 │ 0.355 │ ███▌ │
|
||||
0. │ 1 │ 29 │ 0.42 │ ████▏ │
|
||||
1. │ 1 │ 30 │ 0.477 │ ████▊ │
|
||||
2. │ 1 │ 31 │ 0.529 │ █████▎ │
|
||||
3. │ 1 │ 32 │ 0.576 │ █████▊ │
|
||||
4. │ 1 │ 33 │ 0.618 │ ██████▏ │
|
||||
5. │ 1 │ 34 │ 0.655 │ ██████▌ │
|
||||
6. │ 1 │ 35 │ 0.689 │ ██████▉ │
|
||||
7. │ 1 │ 36 │ 0.719 │ ███████▏ │
|
||||
8. │ 1 │ 37 │ 0.747 │ ███████▍ │
|
||||
9. │ 1 │ 38 │ 0.771 │ ███████▋ │
|
||||
0. │ 1 │ 39 │ 0.793 │ ███████▉ │
|
||||
1. │ 1 │ 40 │ 0.813 │ ████████▏ │
|
||||
2. │ 1 │ 41 │ 0.831 │ ████████▎ │
|
||||
3. │ 1 │ 42 │ 0.848 │ ████████▍ │
|
||||
4. │ 1 │ 43 │ 0.862 │ ████████▌ │
|
||||
5. │ 1 │ 44 │ 0.876 │ ████████▊ │
|
||||
6. │ 1 │ 45 │ 0.888 │ ████████▉ │
|
||||
7. │ 1 │ 46 │ 0.898 │ ████████▉ │
|
||||
8. │ 1 │ 47 │ 0.908 │ █████████ │
|
||||
9. │ 1 │ 48 │ 0.917 │ █████████▏ │
|
||||
0. │ 1 │ 49 │ 0.925 │ █████████▏ │
|
||||
└───────┴──────┴──────────────────────┴────────────┘
|
||||
```
|
||||
@ -0,0 +1,104 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/exponentialTimeDecayedCount
|
||||
sidebar_position: 134
|
||||
title: exponentialTimeDecayedCount
|
||||
---
|
||||
|
||||
## exponentialTimeDecayedCount
|
||||
|
||||
Returns the cumulative exponential decay over a time series at the index `t` in time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
exponentialTimeDecayedCount(x)(t)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `t` — Time. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md), [DateTime](../../data-types/datetime.md), [DateTime64](../../data-types/datetime64.md).
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x` — Half-life period. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned values**
|
||||
|
||||
- Returns the cumulative exponential decay at the given point in time. [Float64](../../data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
value,
|
||||
time,
|
||||
round(exp_smooth, 3),
|
||||
bar(exp_smooth, 0, 20, 50) AS bar
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
(number % 5) = 0 AS value,
|
||||
number AS time,
|
||||
exponentialTimeDecayedCount(10)(time) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
|
||||
FROM numbers(50)
|
||||
);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────────────────────┐
|
||||
1. │ 1 │ 0 │ 1 │ ██▌ │
|
||||
2. │ 0 │ 1 │ 1.905 │ ████▊ │
|
||||
3. │ 0 │ 2 │ 2.724 │ ██████▊ │
|
||||
4. │ 0 │ 3 │ 3.464 │ ████████▋ │
|
||||
5. │ 0 │ 4 │ 4.135 │ ██████████▎ │
|
||||
6. │ 1 │ 5 │ 4.741 │ ███████████▊ │
|
||||
7. │ 0 │ 6 │ 5.29 │ █████████████▏ │
|
||||
8. │ 0 │ 7 │ 5.787 │ ██████████████▍ │
|
||||
9. │ 0 │ 8 │ 6.236 │ ███████████████▌ │
|
||||
10. │ 0 │ 9 │ 6.643 │ ████████████████▌ │
|
||||
11. │ 1 │ 10 │ 7.01 │ █████████████████▌ │
|
||||
12. │ 0 │ 11 │ 7.343 │ ██████████████████▎ │
|
||||
13. │ 0 │ 12 │ 7.644 │ ███████████████████ │
|
||||
14. │ 0 │ 13 │ 7.917 │ ███████████████████▊ │
|
||||
15. │ 0 │ 14 │ 8.164 │ ████████████████████▍ │
|
||||
16. │ 1 │ 15 │ 8.387 │ ████████████████████▉ │
|
||||
17. │ 0 │ 16 │ 8.589 │ █████████████████████▍ │
|
||||
18. │ 0 │ 17 │ 8.771 │ █████████████████████▉ │
|
||||
19. │ 0 │ 18 │ 8.937 │ ██████████████████████▎ │
|
||||
20. │ 0 │ 19 │ 9.086 │ ██████████████████████▋ │
|
||||
21. │ 1 │ 20 │ 9.222 │ ███████████████████████ │
|
||||
22. │ 0 │ 21 │ 9.344 │ ███████████████████████▎ │
|
||||
23. │ 0 │ 22 │ 9.455 │ ███████████████████████▋ │
|
||||
24. │ 0 │ 23 │ 9.555 │ ███████████████████████▉ │
|
||||
25. │ 0 │ 24 │ 9.646 │ ████████████████████████ │
|
||||
26. │ 1 │ 25 │ 9.728 │ ████████████████████████▎ │
|
||||
27. │ 0 │ 26 │ 9.802 │ ████████████████████████▌ │
|
||||
28. │ 0 │ 27 │ 9.869 │ ████████████████████████▋ │
|
||||
29. │ 0 │ 28 │ 9.93 │ ████████████████████████▊ │
|
||||
30. │ 0 │ 29 │ 9.985 │ ████████████████████████▉ │
|
||||
31. │ 1 │ 30 │ 10.035 │ █████████████████████████ │
|
||||
32. │ 0 │ 31 │ 10.08 │ █████████████████████████▏ │
|
||||
33. │ 0 │ 32 │ 10.121 │ █████████████████████████▎ │
|
||||
34. │ 0 │ 33 │ 10.158 │ █████████████████████████▍ │
|
||||
35. │ 0 │ 34 │ 10.191 │ █████████████████████████▍ │
|
||||
36. │ 1 │ 35 │ 10.221 │ █████████████████████████▌ │
|
||||
37. │ 0 │ 36 │ 10.249 │ █████████████████████████▌ │
|
||||
38. │ 0 │ 37 │ 10.273 │ █████████████████████████▋ │
|
||||
39. │ 0 │ 38 │ 10.296 │ █████████████████████████▋ │
|
||||
40. │ 0 │ 39 │ 10.316 │ █████████████████████████▊ │
|
||||
41. │ 1 │ 40 │ 10.334 │ █████████████████████████▊ │
|
||||
42. │ 0 │ 41 │ 10.351 │ █████████████████████████▉ │
|
||||
43. │ 0 │ 42 │ 10.366 │ █████████████████████████▉ │
|
||||
44. │ 0 │ 43 │ 10.379 │ █████████████████████████▉ │
|
||||
45. │ 0 │ 44 │ 10.392 │ █████████████████████████▉ │
|
||||
46. │ 1 │ 45 │ 10.403 │ ██████████████████████████ │
|
||||
47. │ 0 │ 46 │ 10.413 │ ██████████████████████████ │
|
||||
48. │ 0 │ 47 │ 10.422 │ ██████████████████████████ │
|
||||
49. │ 0 │ 48 │ 10.43 │ ██████████████████████████ │
|
||||
50. │ 0 │ 49 │ 10.438 │ ██████████████████████████ │
|
||||
└───────┴──────┴──────────────────────┴────────────────────────────┘
|
||||
```
|
||||
@ -0,0 +1,105 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/exponentialTimeDecayedMax
|
||||
sidebar_position: 135
|
||||
title: exponentialTimeDecayedMax
|
||||
---
|
||||
|
||||
## exponentialTimeDecayedMax
|
||||
|
||||
Returns the maximum of the computed exponentially smoothed moving average at index `t` in time with that at `t-1`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
exponentialTimeDecayedMax(x)(value, timeunit)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Value. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
- `timeunit` — Timeunit. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md), [DateTime](../../data-types/datetime.md), [DateTime64](../../data-types/datetime64.md).
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x` — Half-life period. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned values**
|
||||
|
||||
- Returns the maximum of the exponentially smoothed weighted moving average at `t` and `t-1`. [Float64](../../data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
value,
|
||||
time,
|
||||
round(exp_smooth, 3),
|
||||
bar(exp_smooth, 0, 5, 50) AS bar
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
(number = 0) OR (number >= 25) AS value,
|
||||
number AS time,
|
||||
exponentialTimeDecayedMax(10)(value, time) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
|
||||
FROM numbers(50)
|
||||
);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────┐
|
||||
1. │ 1 │ 0 │ 1 │ ██████████ │
|
||||
2. │ 0 │ 1 │ 0.905 │ █████████ │
|
||||
3. │ 0 │ 2 │ 0.819 │ ████████▏ │
|
||||
4. │ 0 │ 3 │ 0.741 │ ███████▍ │
|
||||
5. │ 0 │ 4 │ 0.67 │ ██████▋ │
|
||||
6. │ 0 │ 5 │ 0.607 │ ██████ │
|
||||
7. │ 0 │ 6 │ 0.549 │ █████▍ │
|
||||
8. │ 0 │ 7 │ 0.497 │ ████▉ │
|
||||
9. │ 0 │ 8 │ 0.449 │ ████▍ │
|
||||
10. │ 0 │ 9 │ 0.407 │ ████ │
|
||||
11. │ 0 │ 10 │ 0.368 │ ███▋ │
|
||||
12. │ 0 │ 11 │ 0.333 │ ███▎ │
|
||||
13. │ 0 │ 12 │ 0.301 │ ███ │
|
||||
14. │ 0 │ 13 │ 0.273 │ ██▋ │
|
||||
15. │ 0 │ 14 │ 0.247 │ ██▍ │
|
||||
16. │ 0 │ 15 │ 0.223 │ ██▏ │
|
||||
17. │ 0 │ 16 │ 0.202 │ ██ │
|
||||
18. │ 0 │ 17 │ 0.183 │ █▊ │
|
||||
19. │ 0 │ 18 │ 0.165 │ █▋ │
|
||||
20. │ 0 │ 19 │ 0.15 │ █▍ │
|
||||
21. │ 0 │ 20 │ 0.135 │ █▎ │
|
||||
22. │ 0 │ 21 │ 0.122 │ █▏ │
|
||||
23. │ 0 │ 22 │ 0.111 │ █ │
|
||||
24. │ 0 │ 23 │ 0.1 │ █ │
|
||||
25. │ 0 │ 24 │ 0.091 │ ▉ │
|
||||
26. │ 1 │ 25 │ 1 │ ██████████ │
|
||||
27. │ 1 │ 26 │ 1 │ ██████████ │
|
||||
28. │ 1 │ 27 │ 1 │ ██████████ │
|
||||
29. │ 1 │ 28 │ 1 │ ██████████ │
|
||||
30. │ 1 │ 29 │ 1 │ ██████████ │
|
||||
31. │ 1 │ 30 │ 1 │ ██████████ │
|
||||
32. │ 1 │ 31 │ 1 │ ██████████ │
|
||||
33. │ 1 │ 32 │ 1 │ ██████████ │
|
||||
34. │ 1 │ 33 │ 1 │ ██████████ │
|
||||
35. │ 1 │ 34 │ 1 │ ██████████ │
|
||||
36. │ 1 │ 35 │ 1 │ ██████████ │
|
||||
37. │ 1 │ 36 │ 1 │ ██████████ │
|
||||
38. │ 1 │ 37 │ 1 │ ██████████ │
|
||||
39. │ 1 │ 38 │ 1 │ ██████████ │
|
||||
40. │ 1 │ 39 │ 1 │ ██████████ │
|
||||
41. │ 1 │ 40 │ 1 │ ██████████ │
|
||||
42. │ 1 │ 41 │ 1 │ ██████████ │
|
||||
43. │ 1 │ 42 │ 1 │ ██████████ │
|
||||
44. │ 1 │ 43 │ 1 │ ██████████ │
|
||||
45. │ 1 │ 44 │ 1 │ ██████████ │
|
||||
46. │ 1 │ 45 │ 1 │ ██████████ │
|
||||
47. │ 1 │ 46 │ 1 │ ██████████ │
|
||||
48. │ 1 │ 47 │ 1 │ ██████████ │
|
||||
49. │ 1 │ 48 │ 1 │ ██████████ │
|
||||
50. │ 1 │ 49 │ 1 │ ██████████ │
|
||||
└───────┴──────┴──────────────────────┴────────────┘
|
||||
```
|
||||
@ -0,0 +1,105 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/exponentialTimeDecayedSum
|
||||
sidebar_position: 136
|
||||
title: exponentialTimeDecayedSum
|
||||
---
|
||||
|
||||
## exponentialTimeDecayedSum
|
||||
|
||||
Returns the sum of exponentially smoothed moving average values of a time series at the index `t` in time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
exponentialTimeDecayedSum(x)(v, t)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `v` — Value. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
- `t` — Time. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md), [DateTime](../../data-types/datetime.md), [DateTime64](../../data-types/datetime64.md).
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x` — Half-life period. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned values**
|
||||
|
||||
- Returns the sum of exponentially smoothed moving average values at the given point in time. [Float64](../../data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
value,
|
||||
time,
|
||||
round(exp_smooth, 3),
|
||||
bar(exp_smooth, 0, 10, 50) AS bar
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
(number = 0) OR (number >= 25) AS value,
|
||||
number AS time,
|
||||
exponentialTimeDecayedSum(10)(value, time) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
|
||||
FROM numbers(50)
|
||||
);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar───────────────────────────────────────────────┐
|
||||
1. │ 1 │ 0 │ 1 │ █████ │
|
||||
2. │ 0 │ 1 │ 0.905 │ ████▌ │
|
||||
3. │ 0 │ 2 │ 0.819 │ ████ │
|
||||
4. │ 0 │ 3 │ 0.741 │ ███▋ │
|
||||
5. │ 0 │ 4 │ 0.67 │ ███▎ │
|
||||
6. │ 0 │ 5 │ 0.607 │ ███ │
|
||||
7. │ 0 │ 6 │ 0.549 │ ██▋ │
|
||||
8. │ 0 │ 7 │ 0.497 │ ██▍ │
|
||||
9. │ 0 │ 8 │ 0.449 │ ██▏ │
|
||||
10. │ 0 │ 9 │ 0.407 │ ██ │
|
||||
11. │ 0 │ 10 │ 0.368 │ █▊ │
|
||||
12. │ 0 │ 11 │ 0.333 │ █▋ │
|
||||
13. │ 0 │ 12 │ 0.301 │ █▌ │
|
||||
14. │ 0 │ 13 │ 0.273 │ █▎ │
|
||||
15. │ 0 │ 14 │ 0.247 │ █▏ │
|
||||
16. │ 0 │ 15 │ 0.223 │ █ │
|
||||
17. │ 0 │ 16 │ 0.202 │ █ │
|
||||
18. │ 0 │ 17 │ 0.183 │ ▉ │
|
||||
19. │ 0 │ 18 │ 0.165 │ ▊ │
|
||||
20. │ 0 │ 19 │ 0.15 │ ▋ │
|
||||
21. │ 0 │ 20 │ 0.135 │ ▋ │
|
||||
22. │ 0 │ 21 │ 0.122 │ ▌ │
|
||||
23. │ 0 │ 22 │ 0.111 │ ▌ │
|
||||
24. │ 0 │ 23 │ 0.1 │ ▌ │
|
||||
25. │ 0 │ 24 │ 0.091 │ ▍ │
|
||||
26. │ 1 │ 25 │ 1.082 │ █████▍ │
|
||||
27. │ 1 │ 26 │ 1.979 │ █████████▉ │
|
||||
28. │ 1 │ 27 │ 2.791 │ █████████████▉ │
|
||||
29. │ 1 │ 28 │ 3.525 │ █████████████████▋ │
|
||||
30. │ 1 │ 29 │ 4.19 │ ████████████████████▉ │
|
||||
31. │ 1 │ 30 │ 4.791 │ ███████████████████████▉ │
|
||||
32. │ 1 │ 31 │ 5.335 │ ██████████████████████████▋ │
|
||||
33. │ 1 │ 32 │ 5.827 │ █████████████████████████████▏ │
|
||||
34. │ 1 │ 33 │ 6.273 │ ███████████████████████████████▎ │
|
||||
35. │ 1 │ 34 │ 6.676 │ █████████████████████████████████▍ │
|
||||
36. │ 1 │ 35 │ 7.041 │ ███████████████████████████████████▏ │
|
||||
37. │ 1 │ 36 │ 7.371 │ ████████████████████████████████████▊ │
|
||||
38. │ 1 │ 37 │ 7.669 │ ██████████████████████████████████████▎ │
|
||||
39. │ 1 │ 38 │ 7.939 │ ███████████████████████████████████████▋ │
|
||||
40. │ 1 │ 39 │ 8.184 │ ████████████████████████████████████████▉ │
|
||||
41. │ 1 │ 40 │ 8.405 │ ██████████████████████████████████████████ │
|
||||
42. │ 1 │ 41 │ 8.605 │ ███████████████████████████████████████████ │
|
||||
43. │ 1 │ 42 │ 8.786 │ ███████████████████████████████████████████▉ │
|
||||
44. │ 1 │ 43 │ 8.95 │ ████████████████████████████████████████████▊ │
|
||||
45. │ 1 │ 44 │ 9.098 │ █████████████████████████████████████████████▍ │
|
||||
46. │ 1 │ 45 │ 9.233 │ ██████████████████████████████████████████████▏ │
|
||||
47. │ 1 │ 46 │ 9.354 │ ██████████████████████████████████████████████▊ │
|
||||
48. │ 1 │ 47 │ 9.464 │ ███████████████████████████████████████████████▎ │
|
||||
49. │ 1 │ 48 │ 9.563 │ ███████████████████████████████████████████████▊ │
|
||||
50. │ 1 │ 49 │ 9.653 │ ████████████████████████████████████████████████▎ │
|
||||
└───────┴──────┴──────────────────────┴───────────────────────────────────────────────────┘
|
||||
```
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/first_value
|
||||
sidebar_position: 7
|
||||
sidebar_position: 137
|
||||
---
|
||||
|
||||
# first_value
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/flamegraph
|
||||
sidebar_position: 110
|
||||
slug: /en/sql-reference/aggregate-functions/reference/flame_graph
|
||||
sidebar_position: 138
|
||||
---
|
||||
|
||||
# flameGraph
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/grouparray
|
||||
sidebar_position: 110
|
||||
sidebar_position: 139
|
||||
---
|
||||
|
||||
# groupArray
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/grouparrayinsertat
|
||||
sidebar_position: 112
|
||||
sidebar_position: 140
|
||||
---
|
||||
|
||||
# groupArrayInsertAt
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/grouparrayintersect
|
||||
sidebar_position: 115
|
||||
sidebar_position: 141
|
||||
---
|
||||
|
||||
# groupArrayIntersect
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/grouparraylast
|
||||
sidebar_position: 110
|
||||
sidebar_position: 142
|
||||
---
|
||||
|
||||
# groupArrayLast
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/grouparraymovingavg
|
||||
sidebar_position: 114
|
||||
sidebar_position: 143
|
||||
---
|
||||
|
||||
# groupArrayMovingAvg
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/grouparraymovingsum
|
||||
sidebar_position: 113
|
||||
sidebar_position: 144
|
||||
---
|
||||
|
||||
# groupArrayMovingSum
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/grouparraysample
|
||||
sidebar_position: 114
|
||||
sidebar_position: 145
|
||||
---
|
||||
|
||||
# groupArraySample
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
---
|
||||
toc_priority: 112
|
||||
---
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/grouparraysorted
|
||||
sidebar_position: 146
|
||||
---
|
||||
|
||||
# groupArraySorted {#groupArraySorted}
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/groupbitand
|
||||
sidebar_position: 125
|
||||
sidebar_position: 147
|
||||
---
|
||||
|
||||
# groupBitAnd
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/groupbitmap
|
||||
sidebar_position: 128
|
||||
sidebar_position: 148
|
||||
---
|
||||
|
||||
# groupBitmap
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/groupbitmapand
|
||||
sidebar_position: 129
|
||||
sidebar_position: 149
|
||||
title: groupBitmapAnd
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/groupbitmapor
|
||||
sidebar_position: 130
|
||||
sidebar_position: 150
|
||||
title: groupBitmapOr
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/groupbitmapxor
|
||||
sidebar_position: 131
|
||||
sidebar_position: 151
|
||||
title: groupBitmapXor
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/groupbitor
|
||||
sidebar_position: 126
|
||||
sidebar_position: 152
|
||||
---
|
||||
|
||||
# groupBitOr
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/groupbitxor
|
||||
sidebar_position: 127
|
||||
sidebar_position: 153
|
||||
---
|
||||
|
||||
# groupBitXor
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/groupuniqarray
|
||||
sidebar_position: 111
|
||||
sidebar_position: 154
|
||||
---
|
||||
|
||||
# groupUniqArray
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/intervalLengthSum
|
||||
sidebar_position: 146
|
||||
sidebar_position: 155
|
||||
sidebar_label: intervalLengthSum
|
||||
title: intervalLengthSum
|
||||
---
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest
|
||||
sidebar_position: 300
|
||||
sidebar_position: 156
|
||||
sidebar_label: kolmogorovSmirnovTest
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/kurtpop
|
||||
sidebar_position: 153
|
||||
sidebar_position: 157
|
||||
---
|
||||
|
||||
# kurtPop
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/kurtsamp
|
||||
sidebar_position: 154
|
||||
sidebar_position: 158
|
||||
---
|
||||
|
||||
# kurtSamp
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/largestTriangleThreeBuckets
|
||||
sidebar_position: 312
|
||||
sidebar_position: 159
|
||||
sidebar_label: largestTriangleThreeBuckets
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/last_value
|
||||
sidebar_position: 8
|
||||
sidebar_position: 160
|
||||
---
|
||||
|
||||
# last_value
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/mannwhitneyutest
|
||||
sidebar_position: 310
|
||||
sidebar_position: 161
|
||||
sidebar_label: mannWhitneyUTest
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/max
|
||||
sidebar_position: 3
|
||||
sidebar_position: 162
|
||||
title: max
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/maxintersections
|
||||
sidebar_position: 360
|
||||
sidebar_position: 163
|
||||
title: maxIntersections
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/maxintersectionsposition
|
||||
sidebar_position: 361
|
||||
sidebar_position: 164
|
||||
title: maxIntersectionsPosition
|
||||
---
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/maxmap
|
||||
sidebar_position: 143
|
||||
sidebar_position: 165
|
||||
---
|
||||
|
||||
# maxMap
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user