diff --git a/.clang-tidy b/.clang-tidy
index 896052915f7..de19059d09e 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -37,7 +37,6 @@ Checks: [
'-cert-oop54-cpp',
'-cert-oop57-cpp',
- '-clang-analyzer-optin.core.EnumCastOutOfRange', # https://github.com/abseil/abseil-cpp/issues/1667
'-clang-analyzer-optin.performance.Padding',
'-clang-analyzer-unix.Malloc',
diff --git a/.editorconfig b/.editorconfig
index 8ac06debb5a..8ecaf9b0267 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -19,3 +19,7 @@ charset = utf-8
indent_style = space
indent_size = 4
trim_trailing_whitespace = true
+
+# Some SQL results have trailing whitespace which is removed by IDEs
+[tests/queries/**.reference]
+trim_trailing_whitespace = false
diff --git a/.github/ISSUE_TEMPLATE/10_question.md b/.github/ISSUE_TEMPLATE/10_question.md
deleted file mode 100644
index 08a05a844e0..00000000000
--- a/.github/ISSUE_TEMPLATE/10_question.md
+++ /dev/null
@@ -1,20 +0,0 @@
----
-name: Question
-about: Ask a question about ClickHouse
-title: ''
-labels: question
-assignees: ''
-
----
-
-> Make sure to check documentation https://clickhouse.com/docs/en/ first. If the question is concise and probably has a short answer, asking it in [community Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-1gh9ds7f4-PgDhJAaF8ad5RbWBAAjzFg) is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse
-
-> If you still prefer GitHub issues, remove all this text and ask your question here.
-
-**Company or project name**
-
-Put your company name or project description here
-
-**Question**
-
-Your question
diff --git a/.github/ISSUE_TEMPLATE/10_question.yaml b/.github/ISSUE_TEMPLATE/10_question.yaml
new file mode 100644
index 00000000000..39d4c27807a
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/10_question.yaml
@@ -0,0 +1,20 @@
+name: Question
+description: Ask a question about ClickHouse
+labels: ["question"]
+body:
+ - type: markdown
+ attributes:
+ value: |
+ > Make sure to check documentation https://clickhouse.com/docs/en/ first. If the question is concise and probably has a short answer, asking it in [community Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-1gh9ds7f4-PgDhJAaF8ad5RbWBAAjzFg) is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse
+ - type: textarea
+ attributes:
+ label: Company or project name
+ description: Put your company name or project description here.
+ validations:
+ required: false
+ - type: textarea
+ attributes:
+ label: Question
+ description: Please put your question here.
+ validations:
+ required: true
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 51a1a6e2df8..e045170561d 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -6,6 +6,7 @@ tests/ci/cancel_and_rerun_workflow_lambda/app.py
-->
### Changelog category (leave one):
- New Feature
+- Experimental Feature
- Improvement
- Performance Improvement
- Backward Incompatible Change
@@ -48,21 +49,18 @@ At a minimum, the following information should be added (but add more as needed)
- [ ] Allow: Stateful tests
- [ ] Allow: Integration Tests
- [ ] Allow: Performance tests
-- [ ] Allow: All NOT Required Checks
+- [ ] Allow: All Builds
- [ ] Allow: batch 1, 2 for multi-batch jobs
- [ ] Allow: batch 3, 4, 5, 6 for multi-batch jobs
---
- [ ] Exclude: Style check
- [ ] Exclude: Fast test
-- [ ] Exclude: Integration Tests
-- [ ] Exclude: Stateless tests
-- [ ] Exclude: Stateful tests
-- [ ] Exclude: Performance tests
- [ ] Exclude: All with ASAN
-- [ ] Exclude: All with Aarch64
- [ ] Exclude: All with TSAN, MSAN, UBSAN, Coverage
+- [ ] Exclude: All with aarch64, release, debug
---
- [ ] Do not test
+- [ ] Woolen Wolfdog
- [ ] Upload binaries for special builds
- [ ] Disable merge-commit
- [ ] Disable CI cache
diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
index b0380b939bb..64c3d2f8342 100644
--- a/.github/workflows/backport_branches.yml
+++ b/.github/workflows/backport_branches.yml
@@ -70,7 +70,7 @@ jobs:
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable_test.yml
with:
- test_name: Compatibility check (amd64)
+ test_name: Compatibility check (release)
runner_type: style-checker
data: ${{ needs.RunConfig.outputs.data }}
CompatibilityCheckAarch64:
@@ -159,33 +159,24 @@ jobs:
############################################################################################
##################################### BUILD REPORTER #######################################
############################################################################################
- BuilderReport:
+ Builds_Report:
# run report check for failed builds to indicate the CI error
- if: ${{ !cancelled() }}
- needs:
- - RunConfig
- - BuilderDebAarch64
- - BuilderDebAsan
- - BuilderDebDebug
- - BuilderDebRelease
- - BuilderDebTsan
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: ClickHouse build check
- runner_type: style-checker-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
- BuilderSpecialReport:
- # run report check for failed builds to indicate the CI error
- if: ${{ !cancelled() }}
- needs:
- - RunConfig
- - BuilderBinDarwin
- - BuilderBinDarwinAarch64
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: ClickHouse special build check
- runner_type: style-checker-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
+ if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
+ needs: [RunConfig, BuilderDebAarch64, BuilderDebAsan, BuilderDebDebug, BuilderDebRelease, BuilderDebTsan, BuilderBinDarwin, BuilderBinDarwinAarch64]
+ runs-on: [self-hosted, style-checker-aarch64]
+ steps:
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ - name: Download reports
+ run: |
+ python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds
+ - name: Builds report
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_tsan package_debug binary_darwin binary_darwin_aarch64
+ - name: Set status
+ run: |
+ python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --post --job-name Builds
############################################################################################
#################################### INSTALL PACKAGES ######################################
############################################################################################
@@ -194,7 +185,7 @@ jobs:
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable_test.yml
with:
- test_name: Install packages (amd64)
+ test_name: Install packages (release)
runner_type: style-checker
data: ${{ needs.RunConfig.outputs.data }}
run_command: |
@@ -204,7 +195,7 @@ jobs:
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable_test.yml
with:
- test_name: Install packages (arm64)
+ test_name: Install packages (aarch64)
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
run_command: |
@@ -256,8 +247,7 @@ jobs:
FinishCheck:
if: ${{ !failure() && !cancelled() }}
needs:
- - BuilderReport
- - BuilderSpecialReport
+ - Builds_Report
- FunctionalStatelessTestAsan
- FunctionalStatefulTestDebug
- StressTestTsan
@@ -273,5 +263,8 @@ jobs:
- name: Finish label
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
- python3 finish_check.py
+ # update mergeable check
+ python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+ # update overall ci report
+ python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
python3 merge_pr.py
diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
new file mode 100644
index 00000000000..3988df3b2b1
--- /dev/null
+++ b/.github/workflows/create_release.yml
@@ -0,0 +1,29 @@
+name: CreateRelease
+
+concurrency:
+ group: release
+
+'on':
+ workflow_dispatch:
+ inputs:
+ sha:
+ description: 'The SHA hash of the commit from which to create the release'
+ required: true
+ type: string
+ type:
+ description: 'The type of release: "new" for a new release or "patch" for a patch release'
+ required: true
+ type: choice
+ options:
+ - new
+ - patch
+
+jobs:
+ Release:
+ runs-on: [self-hosted, style-checker-aarch64]
+ steps:
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ - name: Print greeting
+ run: |
+ python3 ./tests/ci/release.py --commit ${{ inputs.sha }} --type ${{ inputs.type }} --dry-run
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index c2a893a8e99..2a7e6f737ab 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -104,8 +104,8 @@ jobs:
with:
stage: Tests_2
data: ${{ needs.RunConfig.outputs.data }}
- # stage for jobs that do not prohibit merge
Tests_3:
+ # Test_3 should not wait for Test_1/Test_2 and should not be blocked by them on master branch since all jobs need to run there.
needs: [RunConfig, Builds_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
uses: ./.github/workflows/reusable_test_stage.yml
@@ -114,23 +114,14 @@ jobs:
data: ${{ needs.RunConfig.outputs.data }}
################################# Reports #################################
- # Reports should be run even if Builds_1/2 failed - put them separately in wf (not in Tests_1/2)
- Builds_1_Report:
+ # Reports should run even if Builds_1/2 fail - run them separately, not in Tests_1/2/3
+ Builds_Report:
# run report check for failed builds to indicate the CI error
- if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
- needs: [RunConfig, Builds_1]
+ if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
+ needs: [RunConfig, Builds_1, Builds_2]
uses: ./.github/workflows/reusable_test.yml
with:
- test_name: ClickHouse build check
- runner_type: style-checker-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
- Builds_2_Report:
- # run report check for failed builds to indicate the CI error
- if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }}
- needs: [RunConfig, Builds_2]
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: ClickHouse special build check
+ test_name: Builds
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
@@ -164,7 +155,7 @@ jobs:
FinishCheck:
if: ${{ !cancelled() }}
- needs: [RunConfig, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
+ needs: [RunConfig, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Check out repository code
@@ -172,4 +163,4 @@ jobs:
- name: Finish label
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
- python3 finish_check.py
+ python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml
index c8b2452829b..01685ee1f5a 100644
--- a/.github/workflows/merge_queue.yml
+++ b/.github/workflows/merge_queue.yml
@@ -96,20 +96,15 @@ jobs:
stage: Tests_1
data: ${{ needs.RunConfig.outputs.data }}
- ################################# Stage Final #################################
- #
- FinishCheck:
- if: ${{ !failure() && !cancelled() }}
+ CheckReadyForMerge:
+ if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
+ # Test_2 or Test_3 must not have jobs required for Mergeable check
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Tests_1]
runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Check out repository code
uses: ClickHouse/checkout@v1
- - name: Check sync status
+ - name: Check and set merge status
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
- python3 sync_pr.py --status
- - name: Finish label
- run: |
- cd "$GITHUB_WORKSPACE/tests/ci"
- python3 finish_check.py ${{ (contains(needs.*.result, 'failure') && github.event_name == 'merge_group') && '--pipeline-failure' || '' }}
+ python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 7d22554473e..4764e6d3c1a 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -126,16 +126,16 @@ jobs:
with:
stage: Builds_2
data: ${{ needs.RunConfig.outputs.data }}
+ # stage for running non-required checks without being blocked by required checks (Test_1) if corresponding settings is selected
Tests_2:
- needs: [RunConfig, Builds_2]
+ needs: [RunConfig, Builds_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
uses: ./.github/workflows/reusable_test_stage.yml
with:
stage: Tests_2
data: ${{ needs.RunConfig.outputs.data }}
- # stage for jobs that do not prohibit merge
Tests_3:
- needs: [RunConfig, Tests_1, Tests_2]
+ needs: [RunConfig, Builds_1, Tests_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
uses: ./.github/workflows/reusable_test_stage.yml
with:
@@ -143,29 +143,21 @@ jobs:
data: ${{ needs.RunConfig.outputs.data }}
################################# Reports #################################
- # Reports should by run even if Builds_1/2 fail, so put them separately in wf (not in Tests_1/2)
- Builds_1_Report:
+ # Reports should run even if Builds_1/2 fail - run them separately (not in Tests_1/2/3)
+ Builds_Report:
# run report check for failed builds to indicate the CI error
- if: ${{ !cancelled() && needs.StyleCheck.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
- needs: [RunConfig, StyleCheck, Builds_1]
+ if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
+ needs: [RunConfig, StyleCheck, Builds_1, Builds_2]
uses: ./.github/workflows/reusable_test.yml
with:
- test_name: ClickHouse build check
- runner_type: style-checker-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
- Builds_2_Report:
- # run report check for failed builds to indicate the CI error
- if: ${{ !cancelled() && needs.StyleCheck.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }}
- needs: [RunConfig, StyleCheck, Builds_2]
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: ClickHouse special build check
+ test_name: Builds
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
CheckReadyForMerge:
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
- needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2]
+ # Test_2 or Test_3 must not have jobs required for Mergeable check
+ needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1]
runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Check out repository code
@@ -181,7 +173,7 @@ jobs:
#
FinishCheck:
if: ${{ !cancelled() }}
- needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
+ needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Check out repository code
@@ -191,7 +183,7 @@ jobs:
- name: Finish label
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
- python3 finish_check.py
+ python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
#############################################################################################
###################################### JEPSEN TESTS #########################################
@@ -204,8 +196,7 @@ jobs:
concurrency:
group: jepsen
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse Keeper Jepsen') }}
- # jepsen needs binary_release build which is in Builds_2
- needs: [RunConfig, Builds_2]
+ needs: [RunConfig, Builds_1]
uses: ./.github/workflows/reusable_test.yml
with:
test_name: ClickHouse Keeper Jepsen
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
index 4d45c8d8d4b..6bf846d7535 100644
--- a/.github/workflows/release_branches.yml
+++ b/.github/workflows/release_branches.yml
@@ -65,7 +65,7 @@ jobs:
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable_test.yml
with:
- test_name: Compatibility check (amd64)
+ test_name: Compatibility check (release)
runner_type: style-checker
data: ${{ needs.RunConfig.outputs.data }}
CompatibilityCheckAarch64:
@@ -176,35 +176,24 @@ jobs:
############################################################################################
##################################### BUILD REPORTER #######################################
############################################################################################
- BuilderReport:
+ Builds_Report:
# run report check for failed builds to indicate the CI error
- if: ${{ !cancelled() }}
- needs:
- - RunConfig
- - BuilderDebRelease
- - BuilderDebAarch64
- - BuilderDebAsan
- - BuilderDebTsan
- - BuilderDebUBsan
- - BuilderDebMsan
- - BuilderDebDebug
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: ClickHouse build check
- runner_type: style-checker-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
- BuilderSpecialReport:
- # run report check for failed builds to indicate the CI error
- if: ${{ !cancelled() }}
- needs:
- - RunConfig
- - BuilderBinDarwin
- - BuilderBinDarwinAarch64
- uses: ./.github/workflows/reusable_test.yml
- with:
- test_name: ClickHouse special build check
- runner_type: style-checker-aarch64
- data: ${{ needs.RunConfig.outputs.data }}
+ if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
+ needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64, BuilderDebAsan, BuilderDebUBsan, BuilderDebMsan, BuilderDebTsan, BuilderDebDebug, BuilderBinDarwin, BuilderBinDarwinAarch64]
+ runs-on: [self-hosted, style-checker-aarch64]
+ steps:
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ - name: Download reports
+ run: |
+ python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds
+ - name: Builds report
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_msan package_ubsan package_tsan package_debug binary_darwin binary_darwin_aarch64
+ - name: Set status
+ run: |
+ python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --post --job-name Builds
MarkReleaseReady:
if: ${{ !failure() && !cancelled() }}
needs:
@@ -244,7 +233,7 @@ jobs:
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable_test.yml
with:
- test_name: Install packages (amd64)
+ test_name: Install packages (release)
runner_type: style-checker
data: ${{ needs.RunConfig.outputs.data }}
run_command: |
@@ -254,7 +243,7 @@ jobs:
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable_test.yml
with:
- test_name: Install packages (arm64)
+ test_name: Install packages (aarch64)
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
run_command: |
@@ -460,8 +449,7 @@ jobs:
needs:
- DockerServerImage
- DockerKeeperImage
- - BuilderReport
- - BuilderSpecialReport
+ - Builds_Report
- MarkReleaseReady
- FunctionalStatelessTestDebug
- FunctionalStatelessTestRelease
@@ -496,4 +484,7 @@ jobs:
- name: Finish label
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
- python3 finish_check.py
+ # update mergeable check
+ python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+ # update overall ci report
+ python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
diff --git a/.github/workflows/tags_stable.yml b/.github/workflows/tags_stable.yml
index e4fc9f0b1d3..2aa7694bc41 100644
--- a/.github/workflows/tags_stable.yml
+++ b/.github/workflows/tags_stable.yml
@@ -46,9 +46,10 @@ jobs:
./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
./utils/list-versions/update-docker-version.sh
GID=$(id -g "${UID}")
- docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 \
+ # --network=host and CI=1 are required for the S3 access from a container
+ docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \
--volume="${GITHUB_WORKSPACE}:/ClickHouse" clickhouse/style-test \
- /ClickHouse/utils/changelog/changelog.py -v --debug-helpers \
+ /ClickHouse/tests/ci/changelog.py -v --debug-helpers \
--gh-user-or-token="$GITHUB_TOKEN" --jobs=5 \
--output="/ClickHouse/docs/changelogs/${GITHUB_TAG}.md" "${GITHUB_TAG}"
git add "./docs/changelogs/${GITHUB_TAG}.md"
diff --git a/.gitmodules b/.gitmodules
index 28696428e8c..12d865307d8 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -91,13 +91,13 @@
[submodule "contrib/aws"]
path = contrib/aws
url = https://github.com/ClickHouse/aws-sdk-cpp
-[submodule "aws-c-event-stream"]
+[submodule "contrib/aws-c-event-stream"]
path = contrib/aws-c-event-stream
url = https://github.com/awslabs/aws-c-event-stream
-[submodule "aws-c-common"]
+[submodule "contrib/aws-c-common"]
path = contrib/aws-c-common
url = https://github.com/awslabs/aws-c-common.git
-[submodule "aws-checksums"]
+[submodule "contrib/aws-checksums"]
path = contrib/aws-checksums
url = https://github.com/awslabs/aws-checksums
[submodule "contrib/curl"]
@@ -163,7 +163,7 @@
url = https://github.com/xz-mirror/xz
[submodule "contrib/abseil-cpp"]
path = contrib/abseil-cpp
- url = https://github.com/abseil/abseil-cpp
+ url = https://github.com/ClickHouse/abseil-cpp.git
[submodule "contrib/dragonbox"]
path = contrib/dragonbox
url = https://github.com/ClickHouse/dragonbox
diff --git a/.yamllint b/.yamllint
index 9d6550ac960..f144e2d47b1 100644
--- a/.yamllint
+++ b/.yamllint
@@ -13,5 +13,4 @@ rules:
level: warning
comments:
min-spaces-from-content: 1
- document-start:
- present: false
+ document-start: disable
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4891b79e4c7..e2eb65e2967 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
### Table of Contents
+**[ClickHouse release v24.6, 2024-07-01](#246)**
**[ClickHouse release v24.5, 2024-05-30](#245)**
**[ClickHouse release v24.4, 2024-04-30](#244)**
**[ClickHouse release v24.3 LTS, 2024-03-26](#243)**
@@ -8,6 +9,169 @@
# 2024 Changelog
+### ClickHouse release 24.6, 2024-07-01
+
+#### Backward Incompatible Change
+* Enable asynchronous load of databases and tables by default. See the `async_load_databases` in config.xml. While this change is fully compatible, it can introduce a difference in behavior. When `async_load_databases` is false, as in the previous versions, the server will not accept connections until all tables are loaded. When `async_load_databases` is true, as in the new version, the server can accept connections before all the tables are loaded. If a query is made to a table that is not yet loaded, it will wait for the table's loading, which can take considerable time. It can change the behavior of the server if it is part of a large distributed system under a load balancer. In the first case, the load balancer can get a connection refusal and quickly failover to another server. In the second case, the load balancer can connect to a server that is still loading the tables, and the query will have a higher latency. Moreover, if many queries accumulate in the waiting state, it can lead to a "thundering herd" problem when they start processing simultaneously. This can make a difference only for highly loaded distributed backends. You can set the value of `async_load_databases` to false to avoid this problem. [#57695](https://github.com/ClickHouse/ClickHouse/pull/57695) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Setting `replace_long_file_name_to_hash` is enabled by default for `MergeTree` tables. [#64457](https://github.com/ClickHouse/ClickHouse/pull/64457) ([Anton Popov](https://github.com/CurtizJ)). This setting is fully compatible, and no actions needed during upgrade. The new data format is supported from all versions starting from 23.9. After enabling this setting, you can no longer downgrade to a version 23.8 or older.
+* Some invalid queries will fail earlier during parsing. Note: disabled the support for inline KQL expressions (the experimental Kusto language) when they are put into a `kql` table function without a string literal, e.g. `kql(garbage | trash)` instead of `kql('garbage | trash')` or `kql($$garbage | trash$$)`. This feature was introduced unintentionally and should not exist. [#61500](https://github.com/ClickHouse/ClickHouse/pull/61500) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Rework parallel processing in `Ordered` mode of storage `S3Queue`. This PR is backward incompatible for Ordered mode if you used settings `s3queue_processing_threads_num` or `s3queue_total_shards_num`. Setting `s3queue_total_shards_num` is deleted, previously it was allowed to use only under `s3queue_allow_experimental_sharded_mode`, which is now deprecated. A new setting is added - `s3queue_buckets`. [#64349](https://github.com/ClickHouse/ClickHouse/pull/64349) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* New functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` were added. Unlike the existing functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake`, the new functions are compatible with function `generateSnowflakeID`, i.e. they accept the snowflake IDs generated by `generateSnowflakeID` and produce snowflake IDs of the same type as `generateSnowflakeID` (i.e. `UInt64`). Furthermore, the new functions default to the UNIX epoch (aka. 1970-01-01), just like `generateSnowflakeID`. If necessary, a different epoch, e.g. Twitter's/X's epoch 2010-11-04 aka. 1288834974657 msec since UNIX epoch, can be passed. The old conversion functions are deprecated and will be removed after a transition period: to use them regardless, enable setting `allow_deprecated_snowflake_conversion_functions`. [#64948](https://github.com/ClickHouse/ClickHouse/pull/64948) ([Robert Schulze](https://github.com/rschu1ze)).
+
+#### New Feature
+* Allow to store named collections in ClickHouse Keeper. [#64574](https://github.com/ClickHouse/ClickHouse/pull/64574) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Support empty tuples. [#55061](https://github.com/ClickHouse/ClickHouse/pull/55061) ([Amos Bird](https://github.com/amosbird)).
+* Add Hilbert Curve encode and decode functions. [#60156](https://github.com/ClickHouse/ClickHouse/pull/60156) ([Artem Mustafin](https://github.com/Artemmm91)).
+* Add support for index analysis over `hilbertEncode`. [#64662](https://github.com/ClickHouse/ClickHouse/pull/64662) ([Artem Mustafin](https://github.com/Artemmm91)).
+* Added support for reading `LINESTRING` geometry in the WKT format using function `readWKTLineString`. [#62519](https://github.com/ClickHouse/ClickHouse/pull/62519) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Allow to attach parts from a different disk. [#63087](https://github.com/ClickHouse/ClickHouse/pull/63087) ([Unalian](https://github.com/Unalian)).
+* Added new SQL functions `generateSnowflakeID` for generating Twitter-style Snowflake IDs. [#63577](https://github.com/ClickHouse/ClickHouse/pull/63577) ([Danila Puzov](https://github.com/kazalika)).
+* Added `merge_workload` and `mutation_workload` settings to regulate how resources are utilized and shared between merges, mutations and other workloads. [#64061](https://github.com/ClickHouse/ClickHouse/pull/64061) ([Sergei Trifonov](https://github.com/serxa)).
+* Add support for comparing `IPv4` and `IPv6` types using the `=` operator. [#64292](https://github.com/ClickHouse/ClickHouse/pull/64292) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)).
+* Support decimal arguments in binary math functions (pow, atan2, max2, min2, hypot). [#64582](https://github.com/ClickHouse/ClickHouse/pull/64582) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
+* Added SQL functions `parseReadableSize` (along with `OrNull` and `OrZero` variants). [#64742](https://github.com/ClickHouse/ClickHouse/pull/64742) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)).
+* Add server settings `max_table_num_to_throw` and `max_database_num_to_throw` to limit the number of databases or tables on `CREATE` queries. [#64781](https://github.com/ClickHouse/ClickHouse/pull/64781) ([Xu Jia](https://github.com/XuJia0210)).
+* Add `_time` virtual column to file alike storages (s3/file/hdfs/url/azureBlobStorage). [#64947](https://github.com/ClickHouse/ClickHouse/pull/64947) ([Ilya Golshtein](https://github.com/ilejn)).
+* Introduced new functions `base64URLEncode`, `base64URLDecode` and `tryBase64URLDecode`. [#64991](https://github.com/ClickHouse/ClickHouse/pull/64991) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
+* Add new function `editDistanceUTF8`, which calculates the [edit distance](https://en.wikipedia.org/wiki/Edit_distance) between two UTF8 strings. [#65269](https://github.com/ClickHouse/ClickHouse/pull/65269) ([LiuNeng](https://github.com/liuneng1994)).
+* Add `http_response_headers` configuration to support custom response headers in custom HTTP handlers. [#63562](https://github.com/ClickHouse/ClickHouse/pull/63562) ([Grigorii](https://github.com/GSokol)).
+* Added a new table function `loop` to support returning query results in an infinite loop. [#63452](https://github.com/ClickHouse/ClickHouse/pull/63452) ([Sariel](https://github.com/sarielwxm)). This is useful for testing.
+* Introduced two additional columns in the `system.query_log`: `used_privileges` and `missing_privileges`. `used_privileges` is populated with the privileges that were checked during query execution, and `missing_privileges` contains required privileges that are missing. [#64597](https://github.com/ClickHouse/ClickHouse/pull/64597) ([Alexey Katsman](https://github.com/alexkats)).
+* Added a setting `output_format_pretty_display_footer_column_names` which when enabled displays column names at the end of the table for long tables (50 rows by default), with the threshold value for minimum number of rows controlled by `output_format_pretty_display_footer_column_names_min_rows`. [#65144](https://github.com/ClickHouse/ClickHouse/pull/65144) ([Shaun Struwig](https://github.com/Blargian)).
+
+#### Experimental Feature
+* Introduce statistics of type "number of distinct values". [#59357](https://github.com/ClickHouse/ClickHouse/pull/59357) ([Han Fei](https://github.com/hanfei1991)).
+* Support statistics with ReplicatedMergeTree. [#64934](https://github.com/ClickHouse/ClickHouse/pull/64934) ([Han Fei](https://github.com/hanfei1991)).
+* If "replica group" is configured for a `Replicated` database, automatically create a cluster that includes replicas from all groups. [#64312](https://github.com/ClickHouse/ClickHouse/pull/64312) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Add settings `parallel_replicas_custom_key_range_lower` and `parallel_replicas_custom_key_range_upper` to control how parallel replicas with dynamic shards parallelizes queries when using a range filter. [#64604](https://github.com/ClickHouse/ClickHouse/pull/64604) ([josh-hildred](https://github.com/josh-hildred)).
+
+#### Performance Improvement
+* Add the ability to reshuffle rows during insert to optimize for size without violating the order set by `PRIMARY KEY`. It's controlled by the setting `optimize_row_order` (off by default). [#63578](https://github.com/ClickHouse/ClickHouse/pull/63578) ([Igor Markelov](https://github.com/ElderlyPassionFruit)).
+* Add a native parquet reader, which can read parquet binary to ClickHouse Columns directly. It's controlled by the setting `input_format_parquet_use_native_reader` (disabled by default). [#60361](https://github.com/ClickHouse/ClickHouse/pull/60361) ([ZhiHong Zhang](https://github.com/copperybean)).
+* Support partial trivial count optimization when the query filter is able to select exact ranges from merge tree tables. [#60463](https://github.com/ClickHouse/ClickHouse/pull/60463) ([Amos Bird](https://github.com/amosbird)).
+* Reduce max memory usage of multithreaded `INSERT`s by collecting chunks of multiple threads in a single transform. [#61047](https://github.com/ClickHouse/ClickHouse/pull/61047) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Reduce the memory usage when using Azure object storage by using fixed memory allocation, avoiding the allocation of an extra buffer. [#63160](https://github.com/ClickHouse/ClickHouse/pull/63160) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Reduce the number of virtual function calls in `ColumnNullable::size`. [#60556](https://github.com/ClickHouse/ClickHouse/pull/60556) ([HappenLee](https://github.com/HappenLee)).
+* Speedup `splitByRegexp` when the regular expression argument is a single-character. [#62696](https://github.com/ClickHouse/ClickHouse/pull/62696) ([Robert Schulze](https://github.com/rschu1ze)).
+* Speed up aggregation by 8-bit and 16-bit keys by keeping track of the min and max keys used. This allows to reduce the number of cells that need to be verified. [#62746](https://github.com/ClickHouse/ClickHouse/pull/62746) ([Jiebin Sun](https://github.com/jiebinn)).
+* Optimize operator IN when the left hand side is `LowCardinality` and the right is a set of constants. [#64060](https://github.com/ClickHouse/ClickHouse/pull/64060) ([Zhiguo Zhou](https://github.com/ZhiguoZh)).
+* Use a thread pool to initialize and destroy hash tables inside `ConcurrentHashJoin`. [#64241](https://github.com/ClickHouse/ClickHouse/pull/64241) ([Nikita Taranov](https://github.com/nickitat)).
+* Optimized vertical merges in tables with sparse columns. [#64311](https://github.com/ClickHouse/ClickHouse/pull/64311) ([Anton Popov](https://github.com/CurtizJ)).
+* Enabled prefetches of data from remote filesystem during vertical merges. It improves latency of vertical merges in tables with data stored on remote filesystem. [#64314](https://github.com/ClickHouse/ClickHouse/pull/64314) ([Anton Popov](https://github.com/CurtizJ)).
+* Reduce redundant calls to `isDefault` of `ColumnSparse::filter` to improve performance. [#64426](https://github.com/ClickHouse/ClickHouse/pull/64426) ([Jiebin Sun](https://github.com/jiebinn)).
+* Speedup `find_super_nodes` and `find_big_family` keeper-client commands by making multiple asynchronous getChildren requests. [#64628](https://github.com/ClickHouse/ClickHouse/pull/64628) ([Alexander Gololobov](https://github.com/davenger)).
+* Improve function `least`/`greatest` for nullable numberic type arguments. [#64668](https://github.com/ClickHouse/ClickHouse/pull/64668) ([KevinyhZou](https://github.com/KevinyhZou)).
+* Allow merging two consequent filtering steps of a query plan. This improves filter-push-down optimization if the filter condition can be pushed down from the parent step. [#64760](https://github.com/ClickHouse/ClickHouse/pull/64760) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Remove bad optimization in the vertical final implementation and re-enable vertical final algorithm by default. [#64783](https://github.com/ClickHouse/ClickHouse/pull/64783) ([Duc Canh Le](https://github.com/canhld94)).
+* Remove ALIAS nodes from the filter expression. This slightly improves performance for queries with `PREWHERE` (with the new analyzer). [#64793](https://github.com/ClickHouse/ClickHouse/pull/64793) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Re-enable OpenSSL session caching. [#65111](https://github.com/ClickHouse/ClickHouse/pull/65111) ([Robert Schulze](https://github.com/rschu1ze)).
+* Added settings to disable materialization of skip indexes and statistics on inserts (`materialize_skip_indexes_on_insert` and `materialize_statistics_on_insert`). [#64391](https://github.com/ClickHouse/ClickHouse/pull/64391) ([Anton Popov](https://github.com/CurtizJ)).
+* Use the allocated memory size to calculate the row group size and reduce the peak memory of the parquet writer in the single-threaded mode. [#64424](https://github.com/ClickHouse/ClickHouse/pull/64424) ([LiuNeng](https://github.com/liuneng1994)).
+* Improve the iterator of sparse column to reduce call of `size`. [#64497](https://github.com/ClickHouse/ClickHouse/pull/64497) ([Jiebin Sun](https://github.com/jiebinn)).
+* Update condition to use server-side copy for backups to Azure blob storage. [#64518](https://github.com/ClickHouse/ClickHouse/pull/64518) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Optimized memory usage of vertical merges for tables with high number of skip indexes. [#64580](https://github.com/ClickHouse/ClickHouse/pull/64580) ([Anton Popov](https://github.com/CurtizJ)).
+
+#### Improvement
+* `SHOW CREATE TABLE` executed on top of system tables will now show the super handy comment unique for each table which will explain why this table is needed. [#63788](https://github.com/ClickHouse/ClickHouse/pull/63788) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* The second argument (scale) of functions `round()`, `roundBankers()`, `floor()`, `ceil()` and `trunc()` can now be non-const. [#64798](https://github.com/ClickHouse/ClickHouse/pull/64798) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
+* Hot reload storage policy for `Distributed` tables when adding a new disk. [#58285](https://github.com/ClickHouse/ClickHouse/pull/58285) ([Duc Canh Le](https://github.com/canhld94)).
+* Avoid possible deadlock during MergeTree index analysis when scheduling threads in a saturated service. [#59427](https://github.com/ClickHouse/ClickHouse/pull/59427) ([Sean Haynes](https://github.com/seandhaynes)).
+* Several minor corner case fixes to S3 proxy support & tunneling. [#63427](https://github.com/ClickHouse/ClickHouse/pull/63427) ([Arthur Passos](https://github.com/arthurpassos)).
+* Improve io_uring resubmit visibility. Rename profile event `IOUringSQEsResubmits` -> `IOUringSQEsResubmitsAsync` and add a new one `IOUringSQEsResubmitsSync`. [#63699](https://github.com/ClickHouse/ClickHouse/pull/63699) ([Tomer Shafir](https://github.com/tomershafir)).
+* Added a new setting, `metadata_keep_free_space_bytes` to keep free space on the metadata storage disk. [#64128](https://github.com/ClickHouse/ClickHouse/pull/64128) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Add metrics to track the number of directories created and removed by the `plain_rewritable` metadata storage, and the number of entries in the local-to-remote in-memory map. [#64175](https://github.com/ClickHouse/ClickHouse/pull/64175) ([Julia Kartseva](https://github.com/jkartseva)).
+* The query cache now considers identical queries with different settings as different. This increases robustness in cases where different settings (e.g. `limit` or `additional_table_filters`) would affect the query result. [#64205](https://github.com/ClickHouse/ClickHouse/pull/64205) ([Robert Schulze](https://github.com/rschu1ze)).
+* Support the non standard error code `QpsLimitExceeded` in object storage as a retryable error. [#64225](https://github.com/ClickHouse/ClickHouse/pull/64225) ([Sema Checherinda](https://github.com/CheSema)).
+* Forbid converting a MergeTree table to replicated if the zookeeper path for this table already exists. [#64244](https://github.com/ClickHouse/ClickHouse/pull/64244) ([Kirill](https://github.com/kirillgarbar)).
+* Added a new setting `input_format_parquet_prefer_block_bytes` to control the average output block bytes, and modified the default value of `input_format_parquet_max_block_size` to 65409. [#64427](https://github.com/ClickHouse/ClickHouse/pull/64427) ([LiuNeng](https://github.com/liuneng1994)).
+* Allow proxy to be bypassed for hosts specified in `no_proxy` env variable and ClickHouse proxy configuration. [#63314](https://github.com/ClickHouse/ClickHouse/pull/63314) ([Arthur Passos](https://github.com/arthurpassos)).
+* Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)).
+* Settings from the user's config don't affect merges and mutations for `MergeTree` on top of object storage. [#64456](https://github.com/ClickHouse/ClickHouse/pull/64456) ([alesapin](https://github.com/alesapin)).
+* Support the non standard error code `TotalQpsLimitExceeded` in object storage as a retryable error. [#64520](https://github.com/ClickHouse/ClickHouse/pull/64520) ([Sema Checherinda](https://github.com/CheSema)).
+* Updated Advanced Dashboard for both open-source and ClickHouse Cloud versions to include a chart for 'Maximum concurrent network connections'. [#64610](https://github.com/ClickHouse/ClickHouse/pull/64610) ([Thom O'Connor](https://github.com/thomoco)).
+* Improve progress report on `zeros_mt` and `generateRandom`. [#64804](https://github.com/ClickHouse/ClickHouse/pull/64804) ([Raúl Marín](https://github.com/Algunenano)).
+* Add an asynchronous metric `jemalloc.profile.active` to show whether sampling is currently active. This is an activation mechanism in addition to prof.active; both must be active for the calling thread to sample. [#64842](https://github.com/ClickHouse/ClickHouse/pull/64842) ([Unalian](https://github.com/Unalian)).
+* Remove mark of `allow_experimental_join_condition` as important. This mark may have prevented distributed queries in a mixed versions cluster from being executed successfully. [#65008](https://github.com/ClickHouse/ClickHouse/pull/65008) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Added server Asynchronous metrics `DiskGetObjectThrottler*` and `DiskGetObjectThrottler*` reflecting request per second rate limit defined with `s3_max_get_rps` and `s3_max_put_rps` disk settings and currently available number of requests that could be sent without hitting throttling limit on the disk. Metrics are defined for every disk that has a configured limit. [#65050](https://github.com/ClickHouse/ClickHouse/pull/65050) ([Sergei Trifonov](https://github.com/serxa)).
+* Initialize global trace collector for `Poco::ThreadPool` (needed for Keeper, etc). [#65239](https://github.com/ClickHouse/ClickHouse/pull/65239) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add a validation when creating a user with `bcrypt_hash`. [#65242](https://github.com/ClickHouse/ClickHouse/pull/65242) ([Raúl Marín](https://github.com/Algunenano)).
+* Add profile events for number of rows read during/after `PREWHERE`. [#64198](https://github.com/ClickHouse/ClickHouse/pull/64198) ([Nikita Taranov](https://github.com/nickitat)).
+* Print query in `EXPLAIN PLAN` with parallel replicas. [#64298](https://github.com/ClickHouse/ClickHouse/pull/64298) ([vdimir](https://github.com/vdimir)).
+* Rename `allow_deprecated_functions` to `allow_deprecated_error_prone_window_functions`. [#64358](https://github.com/ClickHouse/ClickHouse/pull/64358) ([Raúl Marín](https://github.com/Algunenano)).
+* Respect `max_read_buffer_size` setting for file descriptors as well in the `file` table function. [#64532](https://github.com/ClickHouse/ClickHouse/pull/64532) ([Azat Khuzhin](https://github.com/azat)).
+* Disable transactions for unsupported storages even for materialized views. [#64918](https://github.com/ClickHouse/ClickHouse/pull/64918) ([alesapin](https://github.com/alesapin)).
+* Forbid `QUALIFY` clause in the old analyzer. The old analyzer ignored `QUALIFY`, so it could lead to unexpected data removal in mutations. [#65356](https://github.com/ClickHouse/ClickHouse/pull/65356) ([Dmitry Novik](https://github.com/novikd)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* A bug in Apache ORC library was fixed: Fixed ORC statistics calculation, when writing, for unsigned types on all platforms and Int8 on ARM. [#64563](https://github.com/ClickHouse/ClickHouse/pull/64563) ([Michael Kolupaev](https://github.com/al13n321)).
+* Returned back the behaviour of how ClickHouse works and interprets Tuples in CSV format. This change effectively reverts https://github.com/ClickHouse/ClickHouse/pull/60994 and makes it available only under a few settings: `output_format_csv_serialize_tuple_into_separate_columns`, `input_format_csv_deserialize_separate_columns_into_tuple` and `input_format_csv_try_infer_strings_from_quoted_tuples`. [#65170](https://github.com/ClickHouse/ClickHouse/pull/65170) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fix a permission error where a user in a specific situation can escalate their privileges on the default database without necessary grants. [#64769](https://github.com/ClickHouse/ClickHouse/pull/64769) ([pufit](https://github.com/pufit)).
+* Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Use correct memory alignment for Distinct combinator. Previously, crash could happen because of invalid memory allocation when the combinator was used. [#65379](https://github.com/ClickHouse/ClickHouse/pull/65379) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix crash with `DISTINCT` and window functions. [#64767](https://github.com/ClickHouse/ClickHouse/pull/64767) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fixed 'set' skip index not working with IN and indexHint(). [#62083](https://github.com/ClickHouse/ClickHouse/pull/62083) ([Michael Kolupaev](https://github.com/al13n321)).
+* Support executing function during assignment of parameterized view value. [#63502](https://github.com/ClickHouse/ClickHouse/pull/63502) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fixed parquet memory tracking. [#63584](https://github.com/ClickHouse/ClickHouse/pull/63584) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fixed reading of columns of type `Tuple(Map(LowCardinality(String), String), ...)`. [#63956](https://github.com/ClickHouse/ClickHouse/pull/63956) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix an `Cyclic aliases` error for cyclic aliases of different type (expression and function). [#63993](https://github.com/ClickHouse/ClickHouse/pull/63993) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* This fix will use a proper redefined context with the correct definer for each individual view in the query pipeline. [#64079](https://github.com/ClickHouse/ClickHouse/pull/64079) ([pufit](https://github.com/pufit)).
+* Fix analyzer: "Not found column" error is fixed when using INTERPOLATE. [#64096](https://github.com/ClickHouse/ClickHouse/pull/64096) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix creating backups to S3 buckets with different credentials from the disk containing the file. [#64153](https://github.com/ClickHouse/ClickHouse/pull/64153) ([Antonio Andelic](https://github.com/antonio2368)).
+* The query cache now considers two identical queries against different databases as different. The previous behavior could be used to bypass missing privileges to read from a table. [#64199](https://github.com/ClickHouse/ClickHouse/pull/64199) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix possible abort on uncaught exception in ~WriteBufferFromFileDescriptor in StatusFile. [#64206](https://github.com/ClickHouse/ClickHouse/pull/64206) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix `duplicate alias` error for distributed queries with `ARRAY JOIN`. [#64226](https://github.com/ClickHouse/ClickHouse/pull/64226) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix unexpected accurateCast from string to integer. [#64255](https://github.com/ClickHouse/ClickHouse/pull/64255) ([wudidapaopao](https://github.com/wudidapaopao)).
+* Fixed CNF simplification, in case any OR group contains mutually exclusive atoms. [#64256](https://github.com/ClickHouse/ClickHouse/pull/64256) ([Eduard Karacharov](https://github.com/korowa)).
+* Fix Query Tree size validation. [#64377](https://github.com/ClickHouse/ClickHouse/pull/64377) ([Dmitry Novik](https://github.com/novikd)).
+* Fix `Logical error: Bad cast` for `Buffer` table with `PREWHERE`. [#64388](https://github.com/ClickHouse/ClickHouse/pull/64388) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Prevent recursive logging in `blob_storage_log` when it's stored on object storage. [#64393](https://github.com/ClickHouse/ClickHouse/pull/64393) ([vdimir](https://github.com/vdimir)).
+* Fixed `CREATE TABLE AS` queries for tables with default expressions. [#64455](https://github.com/ClickHouse/ClickHouse/pull/64455) ([Anton Popov](https://github.com/CurtizJ)).
+* Fixed `optimize_read_in_order` behaviour for ORDER BY ... NULLS FIRST / LAST on tables with nullable keys. [#64483](https://github.com/ClickHouse/ClickHouse/pull/64483) ([Eduard Karacharov](https://github.com/korowa)).
+* Fix the `Expression nodes list expected 1 projection names` and `Unknown expression or identifier` errors for queries with aliases to `GLOBAL IN.`. [#64517](https://github.com/ClickHouse/ClickHouse/pull/64517) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix an error `Cannot find column` in distributed queries with constant CTE in the `GROUP BY` key. [#64519](https://github.com/ClickHouse/ClickHouse/pull/64519) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix the crash loop when restoring from backup is blocked by creating an MV with a definer that hasn't been restored yet. [#64595](https://github.com/ClickHouse/ClickHouse/pull/64595) ([pufit](https://github.com/pufit)).
+* Fix the output of function `formatDateTimeInJodaSyntax` when a formatter generates an uneven number of characters and the last character is `0`. For example, `SELECT formatDateTimeInJodaSyntax(toDate('2012-05-29'), 'D')` now correctly returns `150` instead of previously `15`. [#64614](https://github.com/ClickHouse/ClickHouse/pull/64614) ([LiuNeng](https://github.com/liuneng1994)).
+* Do not rewrite aggregation if `-If` combinator is already used. [#64638](https://github.com/ClickHouse/ClickHouse/pull/64638) ([Dmitry Novik](https://github.com/novikd)).
+* Fix type inference for float (in case of small buffer, i.e. `--max_read_buffer_size 1`). [#64641](https://github.com/ClickHouse/ClickHouse/pull/64641) ([Azat Khuzhin](https://github.com/azat)).
+* Fix bug which could lead to non-working TTLs with expressions. [#64694](https://github.com/ClickHouse/ClickHouse/pull/64694) ([alesapin](https://github.com/alesapin)).
+* Fix removing the `WHERE` and `PREWHERE` expressions, which are always true (for the new analyzer). [#64695](https://github.com/ClickHouse/ClickHouse/pull/64695) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed excessive part elimination by token-based text indexes (`ngrambf` , `full_text`) when filtering by result of `startsWith`, `endsWith`, `match`, `multiSearchAny`. [#64720](https://github.com/ClickHouse/ClickHouse/pull/64720) ([Eduard Karacharov](https://github.com/korowa)).
+* Fixes incorrect behaviour of ANSI CSI escaping in the `UTF8::computeWidth` function. [#64756](https://github.com/ClickHouse/ClickHouse/pull/64756) ([Shaun Struwig](https://github.com/Blargian)).
+* Fix a case of incorrect removal of `ORDER BY` / `LIMIT BY` across subqueries. [#64766](https://github.com/ClickHouse/ClickHouse/pull/64766) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix (experimental) unequal join with subqueries for sets which are in the mixed join conditions. [#64775](https://github.com/ClickHouse/ClickHouse/pull/64775) ([lgbo](https://github.com/lgbo-ustc)).
+* Fix crash in a local cache over `plain_rewritable` disk. [#64778](https://github.com/ClickHouse/ClickHouse/pull/64778) ([Julia Kartseva](https://github.com/jkartseva)).
+* Keeper fix: return correct value for `zk_latest_snapshot_size` in `mntr` command. [#64784](https://github.com/ClickHouse/ClickHouse/pull/64784) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix `Cannot find column` in distributed query with `ARRAY JOIN` by `Nested` column. Fixes [#64755](https://github.com/ClickHouse/ClickHouse/issues/64755). [#64801](https://github.com/ClickHouse/ClickHouse/pull/64801) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix memory leak in slru cache policy. [#64803](https://github.com/ClickHouse/ClickHouse/pull/64803) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fixed possible incorrect memory tracking in several kinds of queries: queries that read any data from S3, queries via http protocol, asynchronous inserts. [#64844](https://github.com/ClickHouse/ClickHouse/pull/64844) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix the `Block structure mismatch` error for queries reading with `PREWHERE` from the materialized view when the materialized view has columns of different types than the source table. Fixes [#64611](https://github.com/ClickHouse/ClickHouse/issues/64611). [#64855](https://github.com/ClickHouse/ClickHouse/pull/64855) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix rare crash when table has TTL with subquery + database replicated + parallel replicas + analyzer. It's really rare, but please don't use TTLs with subqueries. [#64858](https://github.com/ClickHouse/ClickHouse/pull/64858) ([alesapin](https://github.com/alesapin)).
+* Fix duplicating `Delete` events in `blob_storage_log` in case of large batch to delete. [#64924](https://github.com/ClickHouse/ClickHouse/pull/64924) ([vdimir](https://github.com/vdimir)).
+* Fixed `Session moved to another server` error from [Zoo]Keeper that might happen after server startup when the config has includes from [Zoo]Keeper. [#64986](https://github.com/ClickHouse/ClickHouse/pull/64986) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix `ALTER MODIFY COMMENT` query that was broken for parameterized VIEWs in https://github.com/ClickHouse/ClickHouse/pull/54211. [#65031](https://github.com/ClickHouse/ClickHouse/pull/65031) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix `host_id` in DatabaseReplicated when `cluster_secure_connection` parameter is enabled. Previously all the connections within the cluster created by DatabaseReplicated were not secure, even if the parameter was enabled. [#65054](https://github.com/ClickHouse/ClickHouse/pull/65054) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fixing the `Not-ready Set` error after the `PREWHERE` optimization for StorageMerge. [#65057](https://github.com/ClickHouse/ClickHouse/pull/65057) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Avoid writing to finalized buffer in File-like storages. [#65063](https://github.com/ClickHouse/ClickHouse/pull/65063) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix possible infinite query duration in case of cyclic aliases. Fixes [#64849](https://github.com/ClickHouse/ClickHouse/issues/64849). [#65081](https://github.com/ClickHouse/ClickHouse/pull/65081) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix the `Unknown expression identifier` error for remote queries with `INTERPOLATE (alias)` (new analyzer). Fixes [#64636](https://github.com/ClickHouse/ClickHouse/issues/64636). [#65090](https://github.com/ClickHouse/ClickHouse/pull/65090) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix pushing arithmetic operations out of aggregation. In the new analyzer, optimization was applied only once. [#65104](https://github.com/ClickHouse/ClickHouse/pull/65104) ([Dmitry Novik](https://github.com/novikd)).
+* Fix aggregate function name rewriting in the new analyzer. [#65110](https://github.com/ClickHouse/ClickHouse/pull/65110) ([Dmitry Novik](https://github.com/novikd)).
+* Respond with 5xx instead of 200 OK in case of receive timeout while reading (parts of) the request body from the client socket. [#65118](https://github.com/ClickHouse/ClickHouse/pull/65118) ([Julian Maicher](https://github.com/jmaicher)).
+* Fix possible crash for hedged requests. [#65206](https://github.com/ClickHouse/ClickHouse/pull/65206) ([Azat Khuzhin](https://github.com/azat)).
+* Fix the bug in Hashed and Hashed_Array dictionary short circuit evaluation, which may read uninitialized number, leading to various errors. [#65256](https://github.com/ClickHouse/ClickHouse/pull/65256) ([jsc0218](https://github.com/jsc0218)).
+* This PR ensures that the type of the constant(IN operator's second parameter) is always visible during the IN operator's type conversion process. Otherwise, losing type information may cause some conversions to fail, such as the conversion from DateTime to Date. This fixes ([#64487](https://github.com/ClickHouse/ClickHouse/issues/64487)). [#65315](https://github.com/ClickHouse/ClickHouse/pull/65315) ([pn](https://github.com/chloro-pn)).
+
+#### Build/Testing/Packaging Improvement
+* Add support for LLVM XRay. [#64592](https://github.com/ClickHouse/ClickHouse/pull/64592) [#64837](https://github.com/ClickHouse/ClickHouse/pull/64837) ([Tomer Shafir](https://github.com/tomershafir)).
+* Unite s3/hdfs/azure storage implementations into a single class working with IObjectStorage. Same for *Cluster, data lakes and Queue storages. [#59767](https://github.com/ClickHouse/ClickHouse/pull/59767) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Refactor data part writer to remove dependencies on MergeTreeData and DataPart. [#63620](https://github.com/ClickHouse/ClickHouse/pull/63620) ([Alexander Gololobov](https://github.com/davenger)).
+* Refactor `KeyCondition` and key analysis to improve PartitionPruner and trivial count optimization. This is separated from [#60463](https://github.com/ClickHouse/ClickHouse/issues/60463) . [#61459](https://github.com/ClickHouse/ClickHouse/pull/61459) ([Amos Bird](https://github.com/amosbird)).
+* Introduce assertions to verify all functions are called with columns of the right size. [#63723](https://github.com/ClickHouse/ClickHouse/pull/63723) ([Raúl Marín](https://github.com/Algunenano)).
+* Make `network` service be required when using the `rc` init script to start the ClickHouse server daemon. [#60650](https://github.com/ClickHouse/ClickHouse/pull/60650) ([Chun-Sheng, Li](https://github.com/peter279k)).
+* Reduce the size of some slow tests. [#64387](https://github.com/ClickHouse/ClickHouse/pull/64387) [#64452](https://github.com/ClickHouse/ClickHouse/pull/64452) ([Raúl Marín](https://github.com/Algunenano)).
+* Replay ZooKeeper logs using keeper-bench. [#62481](https://github.com/ClickHouse/ClickHouse/pull/62481) ([Antonio Andelic](https://github.com/antonio2368)).
+
### ClickHouse release 24.5, 2024-05-30
#### Backward Incompatible Change
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 601cbe7201c..f796e6c4616 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -122,7 +122,7 @@ add_library(global-libs INTERFACE)
include (cmake/sanitize.cmake)
-include (cmake/instrument.cmake)
+include (cmake/xray_instrumentation.cmake)
option(ENABLE_COLORED_BUILD "Enable colors in compiler output" ON)
@@ -319,7 +319,6 @@ endif()
# Disable floating-point expression contraction in order to get consistent floating point calculation results across platforms
set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffp-contract=off")
-# Our built-in unwinder only supports DWARF version up to 4.
set (DEBUG_INFO_FLAGS "-g")
# Disable omit frame pointer compiler optimization using -fno-omit-frame-pointer
@@ -333,15 +332,15 @@ endif()
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
-set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
+set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Og ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
-set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
+set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Og ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} ${COMPILER_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
set (CMAKE_ASM_FLAGS_RELWITHDEBINFO "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
-set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
+set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -Og ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
if (OS_DARWIN)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
@@ -399,7 +398,7 @@ option (ENABLE_GWP_ASAN "Enable Gwp-Asan" ON)
# but GWP-ASan also wants to use mmap frequently,
# and due to a large number of memory mappings,
# it does not work together well.
-if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG"))
+if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") OR SANITIZE)
set(ENABLE_GWP_ASAN OFF)
endif ()
diff --git a/README.md b/README.md
index 73d989210b5..dc253d4db2d 100644
--- a/README.md
+++ b/README.md
@@ -34,20 +34,18 @@ curl https://clickhouse.com/ | sh
Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know.
-* [v24.5 Community Call](https://clickhouse.com/company/events/v24-5-community-release-call) - May 30
+* [v24.6 Community Call](https://clickhouse.com/company/events/v24-6-community-release-call) - Jul 2
## Upcoming Events
Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `` clickhouse `` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
-* [ClickHouse Happy Hour @ Tom's Watch Bar - Los Angeles](https://www.meetup.com/clickhouse-los-angeles-user-group/events/300740584/) - May 22
-* [ClickHouse & Confluent Meetup in Dubai](https://www.meetup.com/clickhouse-dubai-meetup-group/events/299629189/) - May 28
-* [ClickHouse Meetup in Stockholm](https://www.meetup.com/clickhouse-stockholm-user-group/events/299752651/) - Jun 3
-* [ClickHouse Meetup @ Cloudflare - San Francisco](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/300523061/) - Jun 4
-* [ClickHouse (クリックハウス) Meetup Tokyo](https://www.meetup.com/clickhouse-tokyo-user-group/events/300798053/) - Jun 5
+* [AWS Summit in DC](https://clickhouse.com/company/events/2024-06-aws-summit-dc) - Jun 26
* [ClickHouse Meetup in Amsterdam](https://www.meetup.com/clickhouse-netherlands-user-group/events/300781068/) - Jun 27
* [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/300783448/) - Jul 9
+* [ClickHouse Cloud - Live Update Call](https://clickhouse.com/company/events/202407-cloud-update-live) - Jul 9
* [ClickHouse Meetup @ Ramp - New York City](https://www.meetup.com/clickhouse-new-york-user-group/events/300595845/) - Jul 9
+* [AWS Summit in New York](https://clickhouse.com/company/events/2024-07-awssummit-nyc) - Jul 10
* [ClickHouse Meetup @ Klaviyo - Boston](https://www.meetup.com/clickhouse-boston-user-group/events/300907870) - Jul 11
## Recent Recordings
diff --git a/SECURITY.md b/SECURITY.md
index 8635951dc0e..53328b6e16b 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -14,6 +14,7 @@ The following versions of ClickHouse server are currently supported with securit
| Version | Supported |
|:-|:-|
+| 24.6 | ✔️ |
| 24.5 | ✔️ |
| 24.4 | ✔️ |
| 24.3 | ✔️ |
diff --git a/base/base/EnumReflection.h b/base/base/EnumReflection.h
index 4a9de4d17a3..e4e0ef672fd 100644
--- a/base/base/EnumReflection.h
+++ b/base/base/EnumReflection.h
@@ -32,7 +32,7 @@ constexpr void static_for(F && f)
template
struct fmt::formatter : fmt::formatter
{
- constexpr auto format(T value, auto& format_context)
+ constexpr auto format(T value, auto& format_context) const
{
return formatter::format(magic_enum::enum_name(value), format_context);
}
diff --git a/base/base/StringRef.h b/base/base/StringRef.h
index 24af84626de..fc0674b8440 100644
--- a/base/base/StringRef.h
+++ b/base/base/StringRef.h
@@ -12,6 +12,8 @@
#include
#include
#include
+#include
+#include
#include
@@ -376,3 +378,5 @@ namespace PackedZeroTraits
std::ostream & operator<<(std::ostream & os, const StringRef & str);
+
+template<> struct fmt::formatter : fmt::ostream_formatter {};
diff --git a/base/base/getFQDNOrHostName.cpp b/base/base/getFQDNOrHostName.cpp
index 2a4ba8e2e11..6b3da9699b9 100644
--- a/base/base/getFQDNOrHostName.cpp
+++ b/base/base/getFQDNOrHostName.cpp
@@ -6,6 +6,9 @@ namespace
{
std::string getFQDNOrHostNameImpl()
{
+#if defined(OS_DARWIN)
+ return Poco::Net::DNS::hostName();
+#else
try
{
return Poco::Net::DNS::thisHost().name();
@@ -14,6 +17,7 @@ namespace
{
return Poco::Net::DNS::hostName();
}
+#endif
}
}
diff --git a/base/base/wide_integer_to_string.h b/base/base/wide_integer_to_string.h
index c2cbe8d82e3..f703a722afa 100644
--- a/base/base/wide_integer_to_string.h
+++ b/base/base/wide_integer_to_string.h
@@ -62,7 +62,7 @@ struct fmt::formatter>
}
template
- auto format(const wide::integer & value, FormatContext & ctx)
+ auto format(const wide::integer & value, FormatContext & ctx) const
{
return fmt::format_to(ctx.out(), "{}", to_string(value));
}
diff --git a/base/poco/Crypto/src/OpenSSLInitializer.cpp b/base/poco/Crypto/src/OpenSSLInitializer.cpp
index 23447760b47..31798e8dd7e 100644
--- a/base/poco/Crypto/src/OpenSSLInitializer.cpp
+++ b/base/poco/Crypto/src/OpenSSLInitializer.cpp
@@ -23,9 +23,6 @@
#include
#endif
-#if __has_feature(address_sanitizer)
-#include
-#endif
using Poco::RandomInputStream;
using Poco::Thread;
@@ -70,18 +67,12 @@ void OpenSSLInitializer::initialize()
SSL_library_init();
SSL_load_error_strings();
OpenSSL_add_all_algorithms();
-
+
char seed[SEEDSIZE];
RandomInputStream rnd;
rnd.read(seed, sizeof(seed));
- {
-# if __has_feature(address_sanitizer)
- /// Leak sanitizer (part of address sanitizer) thinks that a few bytes of memory in OpenSSL are allocated during but never released.
- __lsan::ScopedDisabler lsan_disabler;
-#endif
- RAND_seed(seed, SEEDSIZE);
- }
-
+ RAND_seed(seed, SEEDSIZE);
+
int nMutexes = CRYPTO_num_locks();
_mutexes = new Poco::FastMutex[nMutexes];
CRYPTO_set_locking_callback(&OpenSSLInitializer::lock);
@@ -89,8 +80,8 @@ void OpenSSLInitializer::initialize()
// https://sourceforge.net/p/poco/bugs/110/
//
// From http://www.openssl.org/docs/crypto/threads.html :
-// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(),
-// then a default implementation is used - on Windows and BeOS this uses the system's
+// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(),
+// then a default implementation is used - on Windows and BeOS this uses the system's
// default thread identifying APIs"
CRYPTO_set_id_callback(&OpenSSLInitializer::id);
CRYPTO_set_dynlock_create_callback(&OpenSSLInitializer::dynlockCreate);
@@ -109,7 +100,7 @@ void OpenSSLInitializer::uninitialize()
CRYPTO_set_locking_callback(0);
CRYPTO_set_id_callback(0);
delete [] _mutexes;
-
+
CONF_modules_free();
}
}
diff --git a/base/poco/Foundation/CMakeLists.txt b/base/poco/Foundation/CMakeLists.txt
index dfb41a33fb1..324a0170bdd 100644
--- a/base/poco/Foundation/CMakeLists.txt
+++ b/base/poco/Foundation/CMakeLists.txt
@@ -213,6 +213,7 @@ target_compile_definitions (_poco_foundation
)
target_include_directories (_poco_foundation SYSTEM PUBLIC "include")
+target_link_libraries (_poco_foundation PRIVATE clickhouse_common_io)
target_link_libraries (_poco_foundation
PRIVATE
diff --git a/base/poco/Foundation/include/Poco/ThreadPool.h b/base/poco/Foundation/include/Poco/ThreadPool.h
index b9506cc5b7f..e2187bfeb66 100644
--- a/base/poco/Foundation/include/Poco/ThreadPool.h
+++ b/base/poco/Foundation/include/Poco/ThreadPool.h
@@ -48,7 +48,13 @@ class Foundation_API ThreadPool
/// from the pool.
{
public:
- ThreadPool(int minCapacity = 2, int maxCapacity = 16, int idleTime = 60, int stackSize = POCO_THREAD_STACK_SIZE);
+ explicit ThreadPool(
+ int minCapacity = 2,
+ int maxCapacity = 16,
+ int idleTime = 60,
+ int stackSize = POCO_THREAD_STACK_SIZE,
+ size_t global_profiler_real_time_period_ns_ = 0,
+ size_t global_profiler_cpu_time_period_ns_ = 0);
/// Creates a thread pool with minCapacity threads.
/// If required, up to maxCapacity threads are created
/// a NoThreadAvailableException exception is thrown.
@@ -56,8 +62,14 @@ public:
/// and more than minCapacity threads are running, the thread
/// is killed. Threads are created with given stack size.
- ThreadPool(
- const std::string & name, int minCapacity = 2, int maxCapacity = 16, int idleTime = 60, int stackSize = POCO_THREAD_STACK_SIZE);
+ explicit ThreadPool(
+ const std::string & name,
+ int minCapacity = 2,
+ int maxCapacity = 16,
+ int idleTime = 60,
+ int stackSize = POCO_THREAD_STACK_SIZE,
+ size_t global_profiler_real_time_period_ns_ = 0,
+ size_t global_profiler_cpu_time_period_ns_ = 0);
/// Creates a thread pool with the given name and minCapacity threads.
/// If required, up to maxCapacity threads are created
/// a NoThreadAvailableException exception is thrown.
@@ -171,6 +183,8 @@ private:
int _serial;
int _age;
int _stackSize;
+ size_t _globalProfilerRealTimePeriodNs;
+ size_t _globalProfilerCPUTimePeriodNs;
ThreadVec _threads;
mutable FastMutex _mutex;
};
diff --git a/base/poco/Foundation/src/ThreadPool.cpp b/base/poco/Foundation/src/ThreadPool.cpp
index 6335ee82b47..f57c81e4128 100644
--- a/base/poco/Foundation/src/ThreadPool.cpp
+++ b/base/poco/Foundation/src/ThreadPool.cpp
@@ -20,6 +20,7 @@
#include "Poco/ErrorHandler.h"
#include
#include
+#include
namespace Poco {
@@ -28,7 +29,11 @@ namespace Poco {
class PooledThread: public Runnable
{
public:
- PooledThread(const std::string& name, int stackSize = POCO_THREAD_STACK_SIZE);
+ explicit PooledThread(
+ const std::string& name,
+ int stackSize = POCO_THREAD_STACK_SIZE,
+ size_t globalProfilerRealTimePeriodNs_ = 0,
+ size_t globalProfilerCPUTimePeriodNs_ = 0);
~PooledThread();
void start();
@@ -51,16 +56,24 @@ private:
Event _targetCompleted;
Event _started;
FastMutex _mutex;
+ size_t _globalProfilerRealTimePeriodNs;
+ size_t _globalProfilerCPUTimePeriodNs;
};
-PooledThread::PooledThread(const std::string& name, int stackSize):
- _idle(true),
- _idleTime(0),
- _pTarget(0),
- _name(name),
+PooledThread::PooledThread(
+ const std::string& name,
+ int stackSize,
+ size_t globalProfilerRealTimePeriodNs_,
+ size_t globalProfilerCPUTimePeriodNs_) :
+ _idle(true),
+ _idleTime(0),
+ _pTarget(0),
+ _name(name),
_thread(name),
- _targetCompleted(false)
+ _targetCompleted(false),
+ _globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
+ _globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
{
poco_assert_dbg (stackSize >= 0);
_thread.setStackSize(stackSize);
@@ -83,7 +96,7 @@ void PooledThread::start()
void PooledThread::start(Thread::Priority priority, Runnable& target)
{
FastMutex::ScopedLock lock(_mutex);
-
+
poco_assert (_pTarget == 0);
_pTarget = ⌖
@@ -109,7 +122,7 @@ void PooledThread::start(Thread::Priority priority, Runnable& target, const std:
}
_thread.setName(fullName);
_thread.setPriority(priority);
-
+
poco_assert (_pTarget == 0);
_pTarget = ⌖
@@ -145,7 +158,7 @@ void PooledThread::join()
void PooledThread::activate()
{
FastMutex::ScopedLock lock(_mutex);
-
+
poco_assert (_idle);
_idle = false;
_targetCompleted.reset();
@@ -155,7 +168,7 @@ void PooledThread::activate()
void PooledThread::release()
{
const long JOIN_TIMEOUT = 10000;
-
+
_mutex.lock();
_pTarget = 0;
_mutex.unlock();
@@ -174,6 +187,10 @@ void PooledThread::release()
void PooledThread::run()
{
+ DB::ThreadStatus thread_status;
+ if (unlikely(_globalProfilerRealTimePeriodNs != 0 || _globalProfilerCPUTimePeriodNs != 0))
+ thread_status.initGlobalProfiler(_globalProfilerRealTimePeriodNs, _globalProfilerCPUTimePeriodNs);
+
_started.set();
for (;;)
{
@@ -220,13 +237,17 @@ void PooledThread::run()
ThreadPool::ThreadPool(int minCapacity,
int maxCapacity,
int idleTime,
- int stackSize):
- _minCapacity(minCapacity),
- _maxCapacity(maxCapacity),
+ int stackSize,
+ size_t globalProfilerRealTimePeriodNs_,
+ size_t globalProfilerCPUTimePeriodNs_) :
+ _minCapacity(minCapacity),
+ _maxCapacity(maxCapacity),
_idleTime(idleTime),
_serial(0),
_age(0),
- _stackSize(stackSize)
+ _stackSize(stackSize),
+ _globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
+ _globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
{
poco_assert (minCapacity >= 1 && maxCapacity >= minCapacity && idleTime > 0);
@@ -243,14 +264,18 @@ ThreadPool::ThreadPool(const std::string& name,
int minCapacity,
int maxCapacity,
int idleTime,
- int stackSize):
+ int stackSize,
+ size_t globalProfilerRealTimePeriodNs_,
+ size_t globalProfilerCPUTimePeriodNs_) :
_name(name),
- _minCapacity(minCapacity),
- _maxCapacity(maxCapacity),
+ _minCapacity(minCapacity),
+ _maxCapacity(maxCapacity),
_idleTime(idleTime),
_serial(0),
_age(0),
- _stackSize(stackSize)
+ _stackSize(stackSize),
+ _globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
+ _globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
{
poco_assert (minCapacity >= 1 && maxCapacity >= minCapacity && idleTime > 0);
@@ -393,15 +418,15 @@ void ThreadPool::housekeep()
ThreadVec activeThreads;
idleThreads.reserve(_threads.size());
activeThreads.reserve(_threads.size());
-
+
for (ThreadVec::iterator it = _threads.begin(); it != _threads.end(); ++it)
{
if ((*it)->idle())
{
if ((*it)->idleTime() < _idleTime)
idleThreads.push_back(*it);
- else
- expiredThreads.push_back(*it);
+ else
+ expiredThreads.push_back(*it);
}
else activeThreads.push_back(*it);
}
@@ -463,7 +488,7 @@ PooledThread* ThreadPool::createThread()
{
std::ostringstream name;
name << _name << "[#" << ++_serial << "]";
- return new PooledThread(name.str(), _stackSize);
+ return new PooledThread(name.str(), _stackSize, _globalProfilerRealTimePeriodNs, _globalProfilerCPUTimePeriodNs);
}
@@ -481,7 +506,7 @@ public:
ThreadPool* pool()
{
FastMutex::ScopedLock lock(_mutex);
-
+
if (!_pPool)
{
_pPool = new ThreadPool("default");
@@ -490,7 +515,7 @@ public:
}
return _pPool;
}
-
+
private:
ThreadPool* _pPool;
FastMutex _mutex;
diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h
index e4037c87927..25dc133fb20 100644
--- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h
+++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h
@@ -17,6 +17,7 @@
#ifndef NetSSL_SSLManager_INCLUDED
#define NetSSL_SSLManager_INCLUDED
+#include
#include
#include "Poco/BasicEvent.h"
@@ -219,6 +220,13 @@ namespace Net
/// Unless initializeClient() has been called, the first call to this method initializes the default Context
/// from the application configuration.
+ Context::Ptr getCustomServerContext(const std::string & name);
+ /// Return custom Context used by the server.
+
+ Context::Ptr setCustomServerContext(const std::string & name, Context::Ptr ctx);
+ /// Set custom Context used by the server.
+ /// Return pointer on inserted Context or on old Context if exists.
+
PrivateKeyPassphraseHandlerPtr serverPassphraseHandler();
/// Returns the configured passphrase handler of the server. If none is set, the method will create a default one
/// from an application configuration.
@@ -258,6 +266,40 @@ namespace Net
static const std::string CFG_SERVER_PREFIX;
static const std::string CFG_CLIENT_PREFIX;
+ static const std::string CFG_PRIV_KEY_FILE;
+ static const std::string CFG_CERTIFICATE_FILE;
+ static const std::string CFG_CA_LOCATION;
+ static const std::string CFG_VER_MODE;
+ static const Context::VerificationMode VAL_VER_MODE;
+ static const std::string CFG_VER_DEPTH;
+ static const int VAL_VER_DEPTH;
+ static const std::string CFG_ENABLE_DEFAULT_CA;
+ static const bool VAL_ENABLE_DEFAULT_CA;
+ static const std::string CFG_CIPHER_LIST;
+ static const std::string CFG_CYPHER_LIST; // for backwards compatibility
+ static const std::string VAL_CIPHER_LIST;
+ static const std::string CFG_PREFER_SERVER_CIPHERS;
+ static const std::string CFG_DELEGATE_HANDLER;
+ static const std::string VAL_DELEGATE_HANDLER;
+ static const std::string CFG_CERTIFICATE_HANDLER;
+ static const std::string VAL_CERTIFICATE_HANDLER;
+ static const std::string CFG_CACHE_SESSIONS;
+ static const std::string CFG_SESSION_ID_CONTEXT;
+ static const std::string CFG_SESSION_CACHE_SIZE;
+ static const std::string CFG_SESSION_TIMEOUT;
+ static const std::string CFG_EXTENDED_VERIFICATION;
+ static const std::string CFG_REQUIRE_TLSV1;
+ static const std::string CFG_REQUIRE_TLSV1_1;
+ static const std::string CFG_REQUIRE_TLSV1_2;
+ static const std::string CFG_DISABLE_PROTOCOLS;
+ static const std::string CFG_DH_PARAMS_FILE;
+ static const std::string CFG_ECDH_CURVE;
+
+#ifdef OPENSSL_FIPS
+ static const std::string CFG_FIPS_MODE;
+ static const bool VAL_FIPS_MODE;
+#endif
+
protected:
static int verifyClientCallback(int ok, X509_STORE_CTX * pStore);
/// The return value of this method defines how errors in
@@ -314,39 +356,7 @@ namespace Net
InvalidCertificateHandlerPtr _ptrClientCertificateHandler;
Poco::FastMutex _mutex;
- static const std::string CFG_PRIV_KEY_FILE;
- static const std::string CFG_CERTIFICATE_FILE;
- static const std::string CFG_CA_LOCATION;
- static const std::string CFG_VER_MODE;
- static const Context::VerificationMode VAL_VER_MODE;
- static const std::string CFG_VER_DEPTH;
- static const int VAL_VER_DEPTH;
- static const std::string CFG_ENABLE_DEFAULT_CA;
- static const bool VAL_ENABLE_DEFAULT_CA;
- static const std::string CFG_CIPHER_LIST;
- static const std::string CFG_CYPHER_LIST; // for backwards compatibility
- static const std::string VAL_CIPHER_LIST;
- static const std::string CFG_PREFER_SERVER_CIPHERS;
- static const std::string CFG_DELEGATE_HANDLER;
- static const std::string VAL_DELEGATE_HANDLER;
- static const std::string CFG_CERTIFICATE_HANDLER;
- static const std::string VAL_CERTIFICATE_HANDLER;
- static const std::string CFG_CACHE_SESSIONS;
- static const std::string CFG_SESSION_ID_CONTEXT;
- static const std::string CFG_SESSION_CACHE_SIZE;
- static const std::string CFG_SESSION_TIMEOUT;
- static const std::string CFG_EXTENDED_VERIFICATION;
- static const std::string CFG_REQUIRE_TLSV1;
- static const std::string CFG_REQUIRE_TLSV1_1;
- static const std::string CFG_REQUIRE_TLSV1_2;
- static const std::string CFG_DISABLE_PROTOCOLS;
- static const std::string CFG_DH_PARAMS_FILE;
- static const std::string CFG_ECDH_CURVE;
-
-#ifdef OPENSSL_FIPS
- static const std::string CFG_FIPS_MODE;
- static const bool VAL_FIPS_MODE;
-#endif
+ std::unordered_map _mapPtrServerContexts;
friend class Poco::SingletonHolder;
friend class Context;
diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h
index 49c12b6b45f..890752c52da 100644
--- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h
+++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h
@@ -235,8 +235,6 @@ namespace Net
/// Note that simply closing a socket is not sufficient
/// to be able to re-use it again.
- Poco::Timespan getMaxTimeout();
-
private:
SecureSocketImpl(const SecureSocketImpl &);
SecureSocketImpl & operator=(const SecureSocketImpl &);
@@ -250,6 +248,9 @@ namespace Net
Session::Ptr _pSession;
friend class SecureStreamSocketImpl;
+
+ Poco::Timespan getMaxTimeoutOrLimit();
+ //// Return max(send, receive) if non zero, otherwise maximum timeout
};
diff --git a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp
index 7f6cc9abcb2..ae04a994786 100644
--- a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp
+++ b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp
@@ -330,27 +330,26 @@ void SSLManager::initDefaultContext(bool server)
else
_ptrDefaultClientContext->disableProtocols(disabledProtocols);
- /// Temporarily disabled during the transition from boringssl to OpenSSL due to tsan issues.
- /// bool cacheSessions = config.getBool(prefix + CFG_CACHE_SESSIONS, false);
- /// if (server)
- /// {
- /// std::string sessionIdContext = config.getString(prefix + CFG_SESSION_ID_CONTEXT, config.getString("application.name", ""));
- /// _ptrDefaultServerContext->enableSessionCache(cacheSessions, sessionIdContext);
- /// if (config.hasProperty(prefix + CFG_SESSION_CACHE_SIZE))
- /// {
- /// int cacheSize = config.getInt(prefix + CFG_SESSION_CACHE_SIZE);
- /// _ptrDefaultServerContext->setSessionCacheSize(cacheSize);
- /// }
- /// if (config.hasProperty(prefix + CFG_SESSION_TIMEOUT))
- /// {
- /// int timeout = config.getInt(prefix + CFG_SESSION_TIMEOUT);
- /// _ptrDefaultServerContext->setSessionTimeout(timeout);
- /// }
- /// }
- /// else
- /// {
- /// _ptrDefaultClientContext->enableSessionCache(cacheSessions);
- /// }
+ bool cacheSessions = config.getBool(prefix + CFG_CACHE_SESSIONS, false);
+ if (server)
+ {
+ std::string sessionIdContext = config.getString(prefix + CFG_SESSION_ID_CONTEXT, config.getString("application.name", ""));
+ _ptrDefaultServerContext->enableSessionCache(cacheSessions, sessionIdContext);
+ if (config.hasProperty(prefix + CFG_SESSION_CACHE_SIZE))
+ {
+ int cacheSize = config.getInt(prefix + CFG_SESSION_CACHE_SIZE);
+ _ptrDefaultServerContext->setSessionCacheSize(cacheSize);
+ }
+ if (config.hasProperty(prefix + CFG_SESSION_TIMEOUT))
+ {
+ int timeout = config.getInt(prefix + CFG_SESSION_TIMEOUT);
+ _ptrDefaultServerContext->setSessionTimeout(timeout);
+ }
+ }
+ else
+ {
+ _ptrDefaultClientContext->enableSessionCache(cacheSessions);
+ }
bool extendedVerification = config.getBool(prefix + CFG_EXTENDED_VERIFICATION, false);
if (server)
_ptrDefaultServerContext->enableExtendedCertificateVerification(extendedVerification);
@@ -429,6 +428,23 @@ void SSLManager::initCertificateHandler(bool server)
}
+Context::Ptr SSLManager::getCustomServerContext(const std::string & name)
+{
+ Poco::FastMutex::ScopedLock lock(_mutex);
+ auto it = _mapPtrServerContexts.find(name);
+ if (it != _mapPtrServerContexts.end())
+ return it->second;
+ return nullptr;
+}
+
+Context::Ptr SSLManager::setCustomServerContext(const std::string & name, Context::Ptr ctx)
+{
+ Poco::FastMutex::ScopedLock lock(_mutex);
+ ctx = _mapPtrServerContexts.insert({name, ctx}).first->second;
+ return ctx;
+}
+
+
Poco::Util::AbstractConfiguration& SSLManager::appConfig()
{
try
diff --git a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp
index efe25f65909..4873d259ae5 100644
--- a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp
+++ b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp
@@ -199,7 +199,7 @@ void SecureSocketImpl::connectSSL(bool performHandshake)
if (performHandshake && _pSocket->getBlocking())
{
int ret;
- Poco::Timespan remaining_time = getMaxTimeout();
+ Poco::Timespan remaining_time = getMaxTimeoutOrLimit();
do
{
RemainingTimeCounter counter(remaining_time);
@@ -302,7 +302,7 @@ int SecureSocketImpl::sendBytes(const void* buffer, int length, int flags)
return rc;
}
- Poco::Timespan remaining_time = getMaxTimeout();
+ Poco::Timespan remaining_time = getMaxTimeoutOrLimit();
do
{
RemainingTimeCounter counter(remaining_time);
@@ -338,7 +338,7 @@ int SecureSocketImpl::receiveBytes(void* buffer, int length, int flags)
return rc;
}
- Poco::Timespan remaining_time = getMaxTimeout();
+ Poco::Timespan remaining_time = getMaxTimeoutOrLimit();
do
{
/// SSL record may consist of several TCP packets,
@@ -372,7 +372,7 @@ int SecureSocketImpl::completeHandshake()
poco_check_ptr (_pSSL);
int rc;
- Poco::Timespan remaining_time = getMaxTimeout();
+ Poco::Timespan remaining_time = getMaxTimeoutOrLimit();
do
{
RemainingTimeCounter counter(remaining_time);
@@ -453,18 +453,29 @@ X509* SecureSocketImpl::peerCertificate() const
return 0;
}
-Poco::Timespan SecureSocketImpl::getMaxTimeout()
+Poco::Timespan SecureSocketImpl::getMaxTimeoutOrLimit()
{
std::lock_guard lock(_mutex);
Poco::Timespan remaining_time = _pSocket->getReceiveTimeout();
Poco::Timespan send_timeout = _pSocket->getSendTimeout();
if (remaining_time < send_timeout)
remaining_time = send_timeout;
+ /// zero SO_SNDTIMEO/SO_RCVTIMEO works as no timeout, let's replicate this
+ ///
+ /// NOTE: we cannot use INT64_MAX (std::numeric_limits::max()),
+ /// since it will be later passed to poll() which accept int timeout, and
+ /// even though poll() accepts milliseconds and Timespan() accepts
+ /// microseconds, let's use smaller maximum value just to avoid some possible
+ /// issues, this should be enough anyway (it is ~24 days).
+ if (remaining_time == 0)
+ remaining_time = Poco::Timespan(std::numeric_limits::max());
return remaining_time;
}
bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time)
{
+ if (remaining_time == 0)
+ return false;
std::lock_guard lock(_mutex);
if (rc <= 0)
{
@@ -475,9 +486,7 @@ bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time)
case SSL_ERROR_WANT_READ:
if (_pSocket->getBlocking())
{
- /// Level-triggered mode of epoll_wait is used, so if SSL_read don't read all available data from socket,
- /// epoll_wait returns true without waiting for new data even if remaining_time == 0
- if (_pSocket->pollImpl(remaining_time, Poco::Net::Socket::SELECT_READ) && remaining_time != 0)
+ if (_pSocket->pollImpl(remaining_time, Poco::Net::Socket::SELECT_READ))
return true;
else
throw Poco::TimeoutException();
@@ -486,13 +495,15 @@ bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time)
case SSL_ERROR_WANT_WRITE:
if (_pSocket->getBlocking())
{
- /// The same as for SSL_ERROR_WANT_READ
- if (_pSocket->pollImpl(remaining_time, Poco::Net::Socket::SELECT_WRITE) && remaining_time != 0)
+ if (_pSocket->pollImpl(remaining_time, Poco::Net::Socket::SELECT_WRITE))
return true;
else
throw Poco::TimeoutException();
}
break;
+ /// NOTE: POCO_EINTR is the same as SSL_ERROR_WANT_READ (at least in
+ /// OpenSSL), so this likely dead code, but let's leave it for
+ /// compatibility with other implementations
case SSL_ERROR_SYSCALL:
return socketError == POCO_EAGAIN || socketError == POCO_EINTR;
default:
diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
index dfbbb66a1e9..bb776fa9506 100644
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@@ -1,12 +1,12 @@
# This variables autochanged by tests/ci/version_helper.py:
-# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
+# NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION,
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54487)
+SET(VERSION_REVISION 54488)
SET(VERSION_MAJOR 24)
-SET(VERSION_MINOR 6)
+SET(VERSION_MINOR 7)
SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 70a1d3a63d47f0be077d67b8deb907230fc7cfb0)
-SET(VERSION_DESCRIBE v24.6.1.1-testing)
-SET(VERSION_STRING 24.6.1.1)
+SET(VERSION_GITHASH aa023477a9265e403982fca5ee29a714db5133d9)
+SET(VERSION_DESCRIBE v24.7.1.1-testing)
+SET(VERSION_STRING 24.7.1.1)
# end of autochange
diff --git a/cmake/instrument.cmake b/cmake/xray_instrumentation.cmake
similarity index 91%
rename from cmake/instrument.cmake
rename to cmake/xray_instrumentation.cmake
index bd2fb4d45fc..661c0575e54 100644
--- a/cmake/instrument.cmake
+++ b/cmake/xray_instrumentation.cmake
@@ -7,7 +7,7 @@ if (NOT ENABLE_XRAY)
return()
endif()
-if (NOT (ARCH_AMD64 AND (OS_LINUX OR OS_FREEBSD)))
+if (NOT (ARCH_AMD64 AND OS_LINUX))
message (STATUS "Not using LLVM XRay, only amd64 Linux or FreeBSD are supported")
return()
endif()
diff --git a/contrib/abseil-cpp b/contrib/abseil-cpp
index 3bd86026c93..a3c4dd3e77f 160000
--- a/contrib/abseil-cpp
+++ b/contrib/abseil-cpp
@@ -1 +1 @@
-Subproject commit 3bd86026c93da5a40006fd53403dff9d5f5e30e3
+Subproject commit a3c4dd3e77f28b526efbb0eb394b72e29c633936
diff --git a/contrib/abseil-cpp-cmake/CMakeLists.txt b/contrib/abseil-cpp-cmake/CMakeLists.txt
index 7372195bb0d..4137547b736 100644
--- a/contrib/abseil-cpp-cmake/CMakeLists.txt
+++ b/contrib/abseil-cpp-cmake/CMakeLists.txt
@@ -1,6 +1,8 @@
set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
set(ABSL_COMMON_INCLUDE_DIRS "${ABSL_ROOT_DIR}")
+# This is a minimized version of the function definition in CMake/AbseilHelpers.cmake
+
#
# Copyright 2017 The Abseil Authors.
#
@@ -16,7 +18,6 @@ set(ABSL_COMMON_INCLUDE_DIRS "${ABSL_ROOT_DIR}")
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
function(absl_cc_library)
cmake_parse_arguments(ABSL_CC_LIB
"DISABLE_INSTALL;PUBLIC;TESTONLY"
@@ -76,6 +77,12 @@ function(absl_cc_library)
add_library(absl::${ABSL_CC_LIB_NAME} ALIAS ${_NAME})
endfunction()
+# The following definitions are an amalgamation of the CMakeLists.txt files in absl/*/
+# To refresh them when upgrading to a new version:
+# - copy them over from upstream
+# - remove calls of 'absl_cc_test'
+# - remove calls of `absl_cc_library` that contain `TESTONLY`
+# - append '${DIR}' to the file definitions
set(DIR ${ABSL_ROOT_DIR}/absl/algorithm)
@@ -102,12 +109,12 @@ absl_cc_library(
absl::algorithm
absl::core_headers
absl::meta
+ absl::nullability
PUBLIC
)
set(DIR ${ABSL_ROOT_DIR}/absl/base)
-# Internal-only target, do not depend on directly.
absl_cc_library(
NAME
atomic_hook
@@ -146,6 +153,18 @@ absl_cc_library(
${ABSL_DEFAULT_COPTS}
)
+absl_cc_library(
+ NAME
+ no_destructor
+ HDRS
+ "${DIR}/no_destructor.h"
+ DEPS
+ absl::config
+ absl::nullability
+ COPTS
+ ${ABSL_DEFAULT_COPTS}
+)
+
absl_cc_library(
NAME
nullability
@@ -305,6 +324,8 @@ absl_cc_library(
${ABSL_DEFAULT_COPTS}
LINKOPTS
${ABSL_DEFAULT_LINKOPTS}
+ $<$:-lrt>
+ $<$:-ladvapi32>
DEPS
absl::atomic_hook
absl::base_internal
@@ -312,6 +333,7 @@ absl_cc_library(
absl::core_headers
absl::dynamic_annotations
absl::log_severity
+ absl::nullability
absl::raw_logging_internal
absl::spinlock_wait
absl::type_traits
@@ -357,6 +379,7 @@ absl_cc_library(
absl::base
absl::config
absl::core_headers
+ absl::nullability
PUBLIC
)
@@ -467,10 +490,11 @@ absl_cc_library(
LINKOPTS
${ABSL_DEFAULT_LINKOPTS}
DEPS
- absl::container_common
absl::common_policy_traits
absl::compare
absl::compressed_tuple
+ absl::config
+ absl::container_common
absl::container_memory
absl::cord
absl::core_headers
@@ -480,7 +504,6 @@ absl_cc_library(
absl::strings
absl::throw_delegate
absl::type_traits
- absl::utility
)
# Internal-only target, do not depend on directly.
@@ -523,7 +546,9 @@ absl_cc_library(
COPTS
${ABSL_DEFAULT_COPTS}
DEPS
+ absl::base_internal
absl::compressed_tuple
+ absl::config
absl::core_headers
absl::memory
absl::span
@@ -548,18 +573,6 @@ absl_cc_library(
PUBLIC
)
-# Internal-only target, do not depend on directly.
-absl_cc_library(
- NAME
- counting_allocator
- HDRS
- "${DIR}/internal/counting_allocator.h"
- COPTS
- ${ABSL_DEFAULT_COPTS}
- DEPS
- absl::config
-)
-
absl_cc_library(
NAME
flat_hash_map
@@ -570,7 +583,7 @@ absl_cc_library(
DEPS
absl::container_memory
absl::core_headers
- absl::hash_function_defaults
+ absl::hash_container_defaults
absl::raw_hash_map
absl::algorithm_container
absl::memory
@@ -586,7 +599,7 @@ absl_cc_library(
${ABSL_DEFAULT_COPTS}
DEPS
absl::container_memory
- absl::hash_function_defaults
+ absl::hash_container_defaults
absl::raw_hash_set
absl::algorithm_container
absl::core_headers
@@ -604,7 +617,7 @@ absl_cc_library(
DEPS
absl::container_memory
absl::core_headers
- absl::hash_function_defaults
+ absl::hash_container_defaults
absl::node_slot_policy
absl::raw_hash_map
absl::algorithm_container
@@ -620,8 +633,9 @@ absl_cc_library(
COPTS
${ABSL_DEFAULT_COPTS}
DEPS
+ absl::container_memory
absl::core_headers
- absl::hash_function_defaults
+ absl::hash_container_defaults
absl::node_slot_policy
absl::raw_hash_set
absl::algorithm_container
@@ -629,6 +643,19 @@ absl_cc_library(
PUBLIC
)
+absl_cc_library(
+ NAME
+ hash_container_defaults
+ HDRS
+ "${DIR}/hash_container_defaults.h"
+ COPTS
+ ${ABSL_DEFAULT_COPTS}
+ DEPS
+ absl::config
+ absl::hash_function_defaults
+ PUBLIC
+)
+
# Internal-only target, do not depend on directly.
absl_cc_library(
NAME
@@ -655,9 +682,11 @@ absl_cc_library(
${ABSL_DEFAULT_COPTS}
DEPS
absl::config
+ absl::container_common
absl::cord
absl::hash
absl::strings
+ absl::type_traits
PUBLIC
)
@@ -703,6 +732,7 @@ absl_cc_library(
absl::base
absl::config
absl::exponential_biased
+ absl::no_destructor
absl::raw_logging_internal
absl::sample_recorder
absl::synchronization
@@ -756,7 +786,9 @@ absl_cc_library(
COPTS
${ABSL_DEFAULT_COPTS}
DEPS
+ absl::config
absl::container_memory
+ absl::core_headers
absl::raw_hash_set
absl::throw_delegate
PUBLIC
@@ -817,6 +849,7 @@ absl_cc_library(
DEPS
absl::config
absl::core_headers
+ absl::debugging_internal
absl::meta
absl::strings
absl::span
@@ -931,6 +964,7 @@ absl_cc_library(
absl::crc32c
absl::config
absl::strings
+ absl::no_destructor
)
set(DIR ${ABSL_ROOT_DIR}/absl/debugging)
@@ -954,6 +988,8 @@ absl_cc_library(
"${DIR}/stacktrace.cc"
COPTS
${ABSL_DEFAULT_COPTS}
+ LINKOPTS
+ $<$:${EXECINFO_LIBRARY}>
DEPS
absl::debugging_internal
absl::config
@@ -980,6 +1016,7 @@ absl_cc_library(
${ABSL_DEFAULT_COPTS}
LINKOPTS
${ABSL_DEFAULT_LINKOPTS}
+ $<$:-ldbghelp>
DEPS
absl::debugging_internal
absl::demangle_internal
@@ -1058,8 +1095,10 @@ absl_cc_library(
demangle_internal
HDRS
"${DIR}/internal/demangle.h"
+ "${DIR}/internal/demangle_rust.h"
SRCS
"${DIR}/internal/demangle.cc"
+ "${DIR}/internal/demangle_rust.cc"
COPTS
${ABSL_DEFAULT_COPTS}
DEPS
@@ -1252,6 +1291,7 @@ absl_cc_library(
absl::strings
absl::synchronization
absl::flat_hash_map
+ absl::no_destructor
)
# Internal-only target, do not depend on directly.
@@ -1283,12 +1323,9 @@ absl_cc_library(
absl_cc_library(
NAME
flags
- SRCS
- "${DIR}/flag.cc"
HDRS
"${DIR}/declare.h"
"${DIR}/flag.h"
- "${DIR}/internal/flag_msvc.inc"
COPTS
${ABSL_DEFAULT_COPTS}
LINKOPTS
@@ -1299,7 +1336,6 @@ absl_cc_library(
absl::flags_config
absl::flags_internal
absl::flags_reflection
- absl::base
absl::core_headers
absl::strings
)
@@ -1379,6 +1415,9 @@ absl_cc_library(
absl::synchronization
)
+############################################################################
+# Unit tests in alphabetical order.
+
set(DIR ${ABSL_ROOT_DIR}/absl/functional)
absl_cc_library(
@@ -1431,6 +1470,18 @@ absl_cc_library(
PUBLIC
)
+absl_cc_library(
+ NAME
+ overload
+ HDRS
+ "${DIR}/overload.h"
+ COPTS
+ ${ABSL_DEFAULT_COPTS}
+ DEPS
+ absl::meta
+ PUBLIC
+)
+
set(DIR ${ABSL_ROOT_DIR}/absl/hash)
absl_cc_library(
@@ -1640,6 +1691,7 @@ absl_cc_library(
absl::log_internal_conditions
absl::log_internal_message
absl::log_internal_strip
+ absl::absl_vlog_is_on
)
absl_cc_library(
@@ -1721,6 +1773,7 @@ absl_cc_library(
absl::log_entry
absl::log_severity
absl::log_sink
+ absl::no_destructor
absl::raw_logging_internal
absl::synchronization
absl::span
@@ -1771,6 +1824,7 @@ absl_cc_library(
LINKOPTS
${ABSL_DEFAULT_LINKOPTS}
DEPS
+ absl::core_headers
absl::log_internal_message
absl::log_internal_nullstream
absl::log_severity
@@ -1876,6 +1930,11 @@ absl_cc_library(
PUBLIC
)
+# Warning: Many linkers will strip the contents of this library because its
+# symbols are only used in a global constructor. A workaround is for clients
+# to link this using $ instead of
+# the plain absl::log_flags.
+# TODO(b/320467376): Implement the equivalent of Bazel's alwayslink=True.
absl_cc_library(
NAME
log_flags
@@ -1897,6 +1956,7 @@ absl_cc_library(
absl::flags
absl::flags_marshalling
absl::strings
+ absl::vlog_config_internal
PUBLIC
)
@@ -1919,6 +1979,7 @@ absl_cc_library(
absl::log_severity
absl::raw_logging_internal
absl::strings
+ absl::vlog_config_internal
)
absl_cc_library(
@@ -1952,6 +2013,7 @@ absl_cc_library(
${ABSL_DEFAULT_LINKOPTS}
DEPS
absl::log_internal_log_impl
+ absl::vlog_is_on
PUBLIC
)
@@ -2064,21 +2126,75 @@ absl_cc_library(
)
absl_cc_library(
- NAME
- log_internal_fnmatch
- SRCS
- "${DIR}/internal/fnmatch.cc"
- HDRS
- "${DIR}/internal/fnmatch.h"
- COPTS
- ${ABSL_DEFAULT_COPTS}
- LINKOPTS
- ${ABSL_DEFAULT_LINKOPTS}
- DEPS
- absl::config
- absl::strings
+ NAME
+ vlog_config_internal
+ SRCS
+ "${DIR}/internal/vlog_config.cc"
+ HDRS
+ "${DIR}/internal/vlog_config.h"
+ COPTS
+ ${ABSL_DEFAULT_COPTS}
+ LINKOPTS
+ ${ABSL_DEFAULT_LINKOPTS}
+ DEPS
+ absl::base
+ absl::config
+ absl::core_headers
+ absl::log_internal_fnmatch
+ absl::memory
+ absl::no_destructor
+ absl::strings
+ absl::synchronization
+ absl::optional
)
+absl_cc_library(
+ NAME
+ absl_vlog_is_on
+ COPTS
+ ${ABSL_DEFAULT_COPTS}
+ LINKOPTS
+ ${ABSL_DEFAULT_LINKOPTS}
+ HDRS
+ "${DIR}/absl_vlog_is_on.h"
+ DEPS
+ absl::vlog_config_internal
+ absl::config
+ absl::core_headers
+ absl::strings
+)
+
+absl_cc_library(
+ NAME
+ vlog_is_on
+ COPTS
+ ${ABSL_DEFAULT_COPTS}
+ LINKOPTS
+ ${ABSL_DEFAULT_LINKOPTS}
+ HDRS
+ "${DIR}/vlog_is_on.h"
+ DEPS
+ absl::absl_vlog_is_on
+)
+
+absl_cc_library(
+ NAME
+ log_internal_fnmatch
+ SRCS
+ "${DIR}/internal/fnmatch.cc"
+ HDRS
+ "${DIR}/internal/fnmatch.h"
+ COPTS
+ ${ABSL_DEFAULT_COPTS}
+ LINKOPTS
+ ${ABSL_DEFAULT_LINKOPTS}
+ DEPS
+ absl::config
+ absl::strings
+)
+
+# Test targets
+
set(DIR ${ABSL_ROOT_DIR}/absl/memory)
absl_cc_library(
@@ -2147,6 +2263,7 @@ absl_cc_library(
COPTS
${ABSL_DEFAULT_COPTS}
DEPS
+ absl::compare
absl::config
absl::core_headers
absl::bits
@@ -2176,6 +2293,8 @@ absl_cc_library(
PUBLIC
)
+set(DIR ${ABSL_ROOT_DIR}/absl/profiling)
+
absl_cc_library(
NAME
sample_recorder
@@ -2188,8 +2307,6 @@ absl_cc_library(
absl::synchronization
)
-set(DIR ${ABSL_ROOT_DIR}/absl/profiling)
-
absl_cc_library(
NAME
exponential_biased
@@ -2265,6 +2382,7 @@ absl_cc_library(
LINKOPTS
${ABSL_DEFAULT_LINKOPTS}
DEPS
+ absl::config
absl::fast_type_id
absl::optional
)
@@ -2336,11 +2454,13 @@ absl_cc_library(
DEPS
absl::config
absl::inlined_vector
+ absl::nullability
absl::random_internal_pool_urbg
absl::random_internal_salted_seed_seq
absl::random_internal_seed_material
absl::random_seed_gen_exception
absl::span
+ absl::string_view
)
# Internal-only target, do not depend on directly.
@@ -2399,6 +2519,7 @@ absl_cc_library(
${ABSL_DEFAULT_COPTS}
LINKOPTS
${ABSL_DEFAULT_LINKOPTS}
+ $<$:-lbcrypt>
DEPS
absl::core_headers
absl::optional
@@ -2658,6 +2779,29 @@ absl_cc_library(
absl::config
)
+# Internal-only target, do not depend on directly.
+absl_cc_library(
+ NAME
+ random_internal_distribution_test_util
+ SRCS
+ "${DIR}/internal/chi_square.cc"
+ "${DIR}/internal/distribution_test_util.cc"
+ HDRS
+ "${DIR}/internal/chi_square.h"
+ "${DIR}/internal/distribution_test_util.h"
+ COPTS
+ ${ABSL_DEFAULT_COPTS}
+ LINKOPTS
+ ${ABSL_DEFAULT_LINKOPTS}
+ DEPS
+ absl::config
+ absl::core_headers
+ absl::raw_logging_internal
+ absl::strings
+ absl::str_format
+ absl::span
+)
+
# Internal-only target, do not depend on directly.
absl_cc_library(
NAME
@@ -2699,6 +2843,8 @@ absl_cc_library(
absl::function_ref
absl::inlined_vector
absl::memory
+ absl::no_destructor
+ absl::nullability
absl::optional
absl::raw_logging_internal
absl::span
@@ -2724,8 +2870,11 @@ absl_cc_library(
absl::base
absl::config
absl::core_headers
+ absl::has_ostream_operator
+ absl::nullability
absl::raw_logging_internal
absl::status
+ absl::str_format
absl::strings
absl::type_traits
absl::utility
@@ -2748,6 +2897,7 @@ absl_cc_library(
absl::base
absl::config
absl::core_headers
+ absl::nullability
absl::throw_delegate
PUBLIC
)
@@ -2762,6 +2912,7 @@ absl_cc_library(
"${DIR}/has_absl_stringify.h"
"${DIR}/internal/damerau_levenshtein_distance.h"
"${DIR}/internal/string_constant.h"
+ "${DIR}/internal/has_absl_stringify.h"
"${DIR}/match.h"
"${DIR}/numbers.h"
"${DIR}/str_cat.h"
@@ -2805,6 +2956,7 @@ absl_cc_library(
absl::endian
absl::int128
absl::memory
+ absl::nullability
absl::raw_logging_internal
absl::throw_delegate
absl::type_traits
@@ -2824,6 +2976,18 @@ absl_cc_library(
PUBLIC
)
+absl_cc_library(
+ NAME
+ has_ostream_operator
+ HDRS
+ "${DIR}/has_ostream_operator.h"
+ COPTS
+ ${ABSL_DEFAULT_COPTS}
+ DEPS
+ absl::config
+ PUBLIC
+)
+
# Internal-only target, do not depend on directly.
absl_cc_library(
NAME
@@ -2855,7 +3019,12 @@ absl_cc_library(
COPTS
${ABSL_DEFAULT_COPTS}
DEPS
+ absl::config
+ absl::core_headers
+ absl::nullability
+ absl::span
absl::str_format_internal
+ absl::string_view
PUBLIC
)
@@ -2886,6 +3055,7 @@ absl_cc_library(
absl::strings
absl::config
absl::core_headers
+ absl::fixed_array
absl::inlined_vector
absl::numeric_representation
absl::type_traits
@@ -2989,6 +3159,7 @@ absl_cc_library(
DEPS
absl::base
absl::config
+ absl::no_destructor
absl::raw_logging_internal
absl::synchronization
)
@@ -3079,6 +3250,7 @@ absl_cc_library(
absl::endian
absl::function_ref
absl::inlined_vector
+ absl::nullability
absl::optional
absl::raw_logging_internal
absl::span
@@ -3246,6 +3418,8 @@ absl_cc_library(
${ABSL_DEFAULT_COPTS}
DEPS
Threads::Threads
+ # TODO(#1495): Use $ once our
+ # minimum CMake version >= 3.24
$<$:-Wl,-framework,CoreFoundation>
)
@@ -3286,8 +3460,8 @@ absl_cc_library(
NAME
bad_any_cast_impl
SRCS
- "${DIR}/bad_any_cast.h"
- "${DIR}/bad_any_cast.cc"
+ "${DIR}/bad_any_cast.h"
+ "${DIR}/bad_any_cast.cc"
COPTS
${ABSL_DEFAULT_COPTS}
DEPS
@@ -3307,6 +3481,7 @@ absl_cc_library(
DEPS
absl::algorithm
absl::core_headers
+ absl::nullability
absl::throw_delegate
absl::type_traits
PUBLIC
@@ -3327,6 +3502,7 @@ absl_cc_library(
absl::config
absl::core_headers
absl::memory
+ absl::nullability
absl::type_traits
absl::utility
PUBLIC
@@ -3389,6 +3565,7 @@ absl_cc_library(
COPTS
${ABSL_DEFAULT_COPTS}
DEPS
+ absl::config
absl::core_headers
absl::type_traits
PUBLIC
diff --git a/contrib/aws b/contrib/aws
index deeaa9e7c5f..1c2946bfcb7 160000
--- a/contrib/aws
+++ b/contrib/aws
@@ -1 +1 @@
-Subproject commit deeaa9e7c5fe690e3dacc4005d7ecfa7a66a32bb
+Subproject commit 1c2946bfcb7f1e3ae0a858de0b59d4f1a7b4ccaf
diff --git a/contrib/cld2 b/contrib/cld2
index bc6d493a2f6..217ba8b8805 160000
--- a/contrib/cld2
+++ b/contrib/cld2
@@ -1 +1 @@
-Subproject commit bc6d493a2f64ed1fc1c4c4b4294a542a04e04217
+Subproject commit 217ba8b8805b41557faadaa47bb6e99f2242eea3
diff --git a/contrib/fmtlib b/contrib/fmtlib
index b6f4ceaed0a..a33701196ad 160000
--- a/contrib/fmtlib
+++ b/contrib/fmtlib
@@ -1 +1 @@
-Subproject commit b6f4ceaed0a0a24ccf575fab6c56dd50ccf6f1a9
+Subproject commit a33701196adfad74917046096bf5a2aa0ab0bb50
diff --git a/contrib/fmtlib-cmake/CMakeLists.txt b/contrib/fmtlib-cmake/CMakeLists.txt
index fe399ddc6e1..6625e411295 100644
--- a/contrib/fmtlib-cmake/CMakeLists.txt
+++ b/contrib/fmtlib-cmake/CMakeLists.txt
@@ -13,7 +13,6 @@ set (SRCS
${FMT_SOURCE_DIR}/include/fmt/core.h
${FMT_SOURCE_DIR}/include/fmt/format.h
${FMT_SOURCE_DIR}/include/fmt/format-inl.h
- ${FMT_SOURCE_DIR}/include/fmt/locale.h
${FMT_SOURCE_DIR}/include/fmt/os.h
${FMT_SOURCE_DIR}/include/fmt/ostream.h
${FMT_SOURCE_DIR}/include/fmt/printf.h
diff --git a/contrib/googletest b/contrib/googletest
index e47544ad31c..a7f443b80b1 160000
--- a/contrib/googletest
+++ b/contrib/googletest
@@ -1 +1 @@
-Subproject commit e47544ad31cb3ceecd04cc13e8fe556f8df9fe0b
+Subproject commit a7f443b80b105f940225332ed3c31f2790092f47
diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt
index b633f0fda50..023fdcf103a 100644
--- a/contrib/jemalloc-cmake/CMakeLists.txt
+++ b/contrib/jemalloc-cmake/CMakeLists.txt
@@ -34,7 +34,7 @@ if (OS_LINUX)
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
- set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
+ set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
else()
set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
endif()
diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c
index e39608998f5..d0a788c5b9f 160000
--- a/contrib/mariadb-connector-c
+++ b/contrib/mariadb-connector-c
@@ -1 +1 @@
-Subproject commit e39608998f5f6944ece9ec61f48e9172ec1de660
+Subproject commit d0a788c5b9fcaca2368d9233770d3ca91ea79f88
diff --git a/contrib/openssl b/contrib/openssl
index f7b8721dfc6..5d81fa7068f 160000
--- a/contrib/openssl
+++ b/contrib/openssl
@@ -1 +1 @@
-Subproject commit f7b8721dfc66abb147f24ca07b9c9d1d64f40f71
+Subproject commit 5d81fa7068fc8c07f4d0997d5b703f3c541a637c
diff --git a/contrib/orc b/contrib/orc
index e24f2c2a3ca..947cebaf943 160000
--- a/contrib/orc
+++ b/contrib/orc
@@ -1 +1 @@
-Subproject commit e24f2c2a3ca0769c96704ab20ad6f512a83ea2ad
+Subproject commit 947cebaf9432d708253ac08dc3012daa6b4ede6f
diff --git a/contrib/re2 b/contrib/re2
index a807e8a3aac..85dd7ad833a 160000
--- a/contrib/re2
+++ b/contrib/re2
@@ -1 +1 @@
-Subproject commit a807e8a3aac2cc33c77b7071efea54fcabe38e0c
+Subproject commit 85dd7ad833a73095ecf3e3baea608ba051bbe2c7
diff --git a/contrib/re2-cmake/CMakeLists.txt b/contrib/re2-cmake/CMakeLists.txt
index f773bc65a69..99d61839b30 100644
--- a/contrib/re2-cmake/CMakeLists.txt
+++ b/contrib/re2-cmake/CMakeLists.txt
@@ -28,16 +28,20 @@ set(RE2_SOURCES
add_library(_re2 ${RE2_SOURCES})
target_include_directories(_re2 PUBLIC "${SRC_DIR}")
target_link_libraries(_re2 PRIVATE
+ absl::absl_check
+ absl::absl_log
absl::base
absl::core_headers
absl::fixed_array
+ absl::flags
absl::flat_hash_map
absl::flat_hash_set
+ absl::hash
absl::inlined_vector
- absl::strings
- absl::str_format
- absl::synchronization
absl::optional
- absl::span)
+ absl::span
+ absl::str_format
+ absl::strings
+ absl::synchronization)
add_library(ch_contrib::re2 ALIAS _re2)
diff --git a/docker/images.json b/docker/images.json
index 7439517379b..716b76ee217 100644
--- a/docker/images.json
+++ b/docker/images.json
@@ -41,8 +41,7 @@
"docker/test/stateless": {
"name": "clickhouse/stateless-test",
"dependent": [
- "docker/test/stateful",
- "docker/test/unit"
+ "docker/test/stateful"
]
},
"docker/test/stateful": {
@@ -122,15 +121,16 @@
"docker/test/base": {
"name": "clickhouse/test-base",
"dependent": [
+ "docker/test/clickbench",
"docker/test/fuzzer",
- "docker/test/libfuzzer",
"docker/test/integration/base",
"docker/test/keeper-jepsen",
+ "docker/test/libfuzzer",
"docker/test/server-jepsen",
"docker/test/sqllogic",
"docker/test/sqltest",
- "docker/test/clickbench",
- "docker/test/stateless"
+ "docker/test/stateless",
+ "docker/test/unit"
]
},
"docker/test/integration/kerberized_hadoop": {
diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile
index b3271d94184..018fe57bf56 100644
--- a/docker/keeper/Dockerfile
+++ b/docker/keeper/Dockerfile
@@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.5.1.1763"
+ARG VERSION="24.6.1.4423"
ARG PACKAGES="clickhouse-keeper"
ARG DIRECT_DOWNLOAD_URLS=""
diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine
index 3f3b880c8f3..a86406e5129 100644
--- a/docker/server/Dockerfile.alpine
+++ b/docker/server/Dockerfile.alpine
@@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.5.1.1763"
+ARG VERSION="24.6.1.4423"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
ARG DIRECT_DOWNLOAD_URLS=""
diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu
index 5fd22ee9b51..25f3273a648 100644
--- a/docker/server/Dockerfile.ubuntu
+++ b/docker/server/Dockerfile.ubuntu
@@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="24.5.1.1763"
+ARG VERSION="24.6.1.4423"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
#docker-official-library:off
diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index b8f967ed9c2..6191aeaf304 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -208,6 +208,7 @@ handle SIGPIPE nostop noprint pass
handle SIGTERM nostop noprint pass
handle SIGUSR1 nostop noprint pass
handle SIGUSR2 nostop noprint pass
+handle SIGSEGV nostop pass
handle SIG$RTMIN nostop noprint pass
info signals
continue
diff --git a/docker/test/stateless/attach_gdb.lib b/docker/test/stateless/attach_gdb.lib
index d288288bb17..eb54f920b98 100644
--- a/docker/test/stateless/attach_gdb.lib
+++ b/docker/test/stateless/attach_gdb.lib
@@ -20,6 +20,7 @@ handle SIGPIPE nostop noprint pass
handle SIGTERM nostop noprint pass
handle SIGUSR1 nostop noprint pass
handle SIGUSR2 nostop noprint pass
+handle SIGSEGV nostop pass
handle SIG$RTMIN nostop noprint pass
info signals
continue
diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 4d2c2e6f466..7d6499cef5e 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -253,8 +253,8 @@ function run_tests()
try_run_with_retry 10 clickhouse-client -q "insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')"
set +e
- clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
- --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
+ timeout -s TERM --preserve-status 120m clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
+ --no-drop-if-fail --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
| ts '%Y-%m-%d %H:%M:%S' \
| tee -a test_output/test_result.txt
set -e
@@ -285,7 +285,7 @@ stop_logs_replication
# Try to get logs while server is running
failed_to_save_logs=0
-for table in query_log zookeeper_log trace_log transactions_info_log metric_log
+for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log error_log
do
err=$(clickhouse-client -q "select * from system.$table into outfile '/test_output/$table.tsv.gz' format TSVWithNamesAndTypes")
echo "$err"
@@ -339,7 +339,7 @@ if [ $failed_to_save_logs -ne 0 ]; then
# directly
# - even though ci auto-compress some files (but not *.tsv) it does this only
# for files >64MB, we want this files to be compressed explicitly
- for table in query_log zookeeper_log trace_log transactions_info_log metric_log
+ for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log error_log
do
clickhouse-local "$data_path_config" --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||:
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
@@ -379,6 +379,10 @@ fi
tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
+rm -rf /var/lib/clickhouse/data/system/*/
+tar -chf /test_output/store.tar /var/lib/clickhouse/store ||:
+tar -chf /test_output/metadata.tar /var/lib/clickhouse/metadata/*.sql ||:
+
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
rg -Fa "" /var/log/clickhouse-server/clickhouse-server1.log ||:
rg -Fa "" /var/log/clickhouse-server/clickhouse-server2.log ||:
diff --git a/docker/test/stateless/stress_tests.lib b/docker/test/stateless/stress_tests.lib
index 3b6ad244c82..c069ccbdd8d 100644
--- a/docker/test/stateless/stress_tests.lib
+++ b/docker/test/stateless/stress_tests.lib
@@ -89,10 +89,6 @@ function configure()
# since we run clickhouse from root
sudo chown root: /var/lib/clickhouse
- # Set more frequent update period of asynchronous metrics to more frequently update information about real memory usage (less chance of OOM).
- echo "1" \
- > /etc/clickhouse-server/config.d/asynchronous_metrics_update_period_s.xml
-
local total_mem
total_mem=$(awk '/MemTotal/ { print $(NF-1) }' /proc/meminfo) # KiB
total_mem=$(( total_mem*1024 )) # bytes
diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile
index cb29185f068..7cd712b73f6 100644
--- a/docker/test/style/Dockerfile
+++ b/docker/test/style/Dockerfile
@@ -10,14 +10,15 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
aspell \
curl \
- git \
- gh \
file \
+ gh \
+ git \
libxml2-utils \
+ locales \
moreutils \
python3-pip \
yamllint \
- locales \
+ zstd \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
@@ -30,10 +31,13 @@ RUN pip3 install \
mypy==1.8.0 \
pylint==3.1.0 \
python-magic==0.4.24 \
+ flake8==4.0.1 \
requests \
thefuzz \
+ tqdm==4.66.4 \
types-requests \
unidiff \
+ jwt \
&& rm -rf /root/.cache/pip
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8
diff --git a/docker/test/style/run.sh b/docker/test/style/run.sh
index cc6cb292b66..64803191532 100755
--- a/docker/test/style/run.sh
+++ b/docker/test/style/run.sh
@@ -9,6 +9,8 @@ echo "Check style" | ts
./check-style -n |& tee /test_output/style_output.txt
echo "Check python formatting with black" | ts
./check-black -n |& tee /test_output/black_output.txt
+echo "Check python with flake8" | ts
+./check-flake8 |& tee /test_output/flake8_output.txt
echo "Check python type hinting with mypy" | ts
./check-mypy -n |& tee /test_output/mypy_output.txt
echo "Check typos" | ts
diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile
index cf5ba1eec7f..af44dc930b2 100644
--- a/docker/test/unit/Dockerfile
+++ b/docker/test/unit/Dockerfile
@@ -1,9 +1,7 @@
# rebuild in #33610
# docker build -t clickhouse/unit-test .
ARG FROM_TAG=latest
-FROM clickhouse/stateless-test:$FROM_TAG
-
-RUN apt-get install gdb
+FROM clickhouse/test-base:$FROM_TAG
COPY run.sh /
CMD ["/bin/bash", "/run.sh"]
diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh
index 1f2cc9903b2..a4c4c75e5b3 100644
--- a/docker/test/upgrade/run.sh
+++ b/docker/test/upgrade/run.sh
@@ -25,7 +25,8 @@ azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
./setup_minio.sh stateless # to have a proper environment
echo "Get previous release tag"
-previous_release_tag=$(dpkg --info package_folder/clickhouse-client*.deb | grep "Version: " | awk '{print $2}' | cut -f1 -d'+' | get_previous_release_tag)
+# shellcheck disable=SC2016
+previous_release_tag=$(dpkg-deb --showformat='${Version}' --show package_folder/clickhouse-client*.deb | get_previous_release_tag)
echo $previous_release_tag
echo "Clone previous release repository"
diff --git a/docs/changelogs/v23.8.15.35-lts.md b/docs/changelogs/v23.8.15.35-lts.md
new file mode 100644
index 00000000000..bab5c507fe8
--- /dev/null
+++ b/docs/changelogs/v23.8.15.35-lts.md
@@ -0,0 +1,40 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v23.8.15.35-lts (060ff8e813a) FIXME as compared to v23.8.14.6-lts (967e51c1d6b)
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#63621](https://github.com/ClickHouse/ClickHouse/issues/63621): The Dockerfile is reviewed by the docker official library in https://github.com/docker-library/official-images/pull/15846. [#63400](https://github.com/ClickHouse/ClickHouse/pull/63400) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#65153](https://github.com/ClickHouse/ClickHouse/issues/65153): Decrease the `unit-test` image a few times. [#65102](https://github.com/ClickHouse/ClickHouse/pull/65102) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Backported in [#64422](https://github.com/ClickHouse/ClickHouse/issues/64422): Fixes [#59989](https://github.com/ClickHouse/ClickHouse/issues/59989): runs init scripts when force-enabled or when no database exists, rather than the inverse. [#59991](https://github.com/ClickHouse/ClickHouse/pull/59991) ([jktng](https://github.com/jktng)).
+* Backported in [#64016](https://github.com/ClickHouse/ClickHouse/issues/64016): Fix "Invalid storage definition in metadata file" for parameterized views. [#60708](https://github.com/ClickHouse/ClickHouse/pull/60708) ([Azat Khuzhin](https://github.com/azat)).
+* Backported in [#63456](https://github.com/ClickHouse/ClickHouse/issues/63456): Fix the issue where the function `addDays` (and similar functions) reports an error when the first parameter is `DateTime64`. [#61561](https://github.com/ClickHouse/ClickHouse/pull/61561) ([Shuai li](https://github.com/loneylee)).
+* Backported in [#63289](https://github.com/ClickHouse/ClickHouse/issues/63289): Fix crash with untuple and unresolved lambda. [#63131](https://github.com/ClickHouse/ClickHouse/pull/63131) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#63512](https://github.com/ClickHouse/ClickHouse/issues/63512): Fix `X-ClickHouse-Timezone` header returning wrong timezone when using `session_timezone` as query level setting. [#63377](https://github.com/ClickHouse/ClickHouse/pull/63377) ([Andrey Zvonov](https://github.com/zvonand)).
+* Backported in [#63902](https://github.com/ClickHouse/ClickHouse/issues/63902): `query_plan_remove_redundant_distinct` can break queries with WINDOW FUNCTIONS (with `allow_experimental_analyzer` is on). Fixes [#62820](https://github.com/ClickHouse/ClickHouse/issues/62820). [#63776](https://github.com/ClickHouse/ClickHouse/pull/63776) ([Igor Nikonov](https://github.com/devcrafter)).
+* Backported in [#64104](https://github.com/ClickHouse/ClickHouse/issues/64104): Deserialize untrusted binary inputs in a safer way. [#64024](https://github.com/ClickHouse/ClickHouse/pull/64024) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#64265](https://github.com/ClickHouse/ClickHouse/issues/64265): Prevent LOGICAL_ERROR on CREATE TABLE as MaterializedView. [#64174](https://github.com/ClickHouse/ClickHouse/pull/64174) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#64867](https://github.com/ClickHouse/ClickHouse/issues/64867): Fixed memory possible incorrect memory tracking in several kinds of queries: queries that read any data from S3, queries via http protocol, asynchronous inserts. [#64844](https://github.com/ClickHouse/ClickHouse/pull/64844) ([Anton Popov](https://github.com/CurtizJ)).
+
+#### NO CL CATEGORY
+
+* Backported in [#63704](https://github.com/ClickHouse/ClickHouse/issues/63704):. [#63415](https://github.com/ClickHouse/ClickHouse/pull/63415) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+
+#### NO CL ENTRY
+
+* NO CL ENTRY: 'Installation test has wrong check_state'. [#63994](https://github.com/ClickHouse/ClickHouse/pull/63994) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#63343](https://github.com/ClickHouse/ClickHouse/issues/63343): The commit url has different pattern. [#63331](https://github.com/ClickHouse/ClickHouse/pull/63331) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#63965](https://github.com/ClickHouse/ClickHouse/issues/63965): fix 02124_insert_deduplication_token_multiple_blocks. [#63950](https://github.com/ClickHouse/ClickHouse/pull/63950) ([Han Fei](https://github.com/hanfei1991)).
+* Backported in [#64043](https://github.com/ClickHouse/ClickHouse/issues/64043): Do not create new release in release branch automatically. [#64039](https://github.com/ClickHouse/ClickHouse/pull/64039) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Pin requests to fix the integration tests. [#65183](https://github.com/ClickHouse/ClickHouse/pull/65183) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
diff --git a/docs/changelogs/v24.1.6.52-stable.md b/docs/changelogs/v24.1.6.52-stable.md
new file mode 100644
index 00000000000..341561e9a64
--- /dev/null
+++ b/docs/changelogs/v24.1.6.52-stable.md
@@ -0,0 +1,45 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.1.6.52-stable (fa09f677bc9) FIXME as compared to v24.1.5.6-stable (7f67181ff31)
+
+#### Improvement
+* Backported in [#60292](https://github.com/ClickHouse/ClickHouse/issues/60292): Copy S3 file GCP fallback to buffer copy in case GCP returned `Internal Error` with `GATEWAY_TIMEOUT` HTTP error code. [#60164](https://github.com/ClickHouse/ClickHouse/pull/60164) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#60832](https://github.com/ClickHouse/ClickHouse/issues/60832): Update tzdata to 2024a. [#60768](https://github.com/ClickHouse/ClickHouse/pull/60768) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Backported in [#60413](https://github.com/ClickHouse/ClickHouse/issues/60413): Fix segmentation fault in KQL parser when the input query exceeds the `max_query_size`. Also re-enable the KQL dialect. Fixes [#59036](https://github.com/ClickHouse/ClickHouse/issues/59036) and [#59037](https://github.com/ClickHouse/ClickHouse/issues/59037). [#59626](https://github.com/ClickHouse/ClickHouse/pull/59626) ([Yong Wang](https://github.com/kashwy)).
+* Backported in [#60074](https://github.com/ClickHouse/ClickHouse/issues/60074): Fix error `Read beyond last offset` for `AsynchronousBoundedReadBuffer`. [#59630](https://github.com/ClickHouse/ClickHouse/pull/59630) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Backported in [#60299](https://github.com/ClickHouse/ClickHouse/issues/60299): Fix having neigher acked nor nacked messages. If exception happens during read-write phase, messages will be nacked. [#59775](https://github.com/ClickHouse/ClickHouse/pull/59775) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#60066](https://github.com/ClickHouse/ClickHouse/issues/60066): Fix optimize_uniq_to_count removing the column alias. [#60026](https://github.com/ClickHouse/ClickHouse/pull/60026) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#60638](https://github.com/ClickHouse/ClickHouse/issues/60638): Fixed a bug in parallel optimization for queries with `FINAL`, which could give an incorrect result in rare cases. [#60041](https://github.com/ClickHouse/ClickHouse/pull/60041) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#60177](https://github.com/ClickHouse/ClickHouse/issues/60177): Fix cosineDistance crash with Nullable. [#60150](https://github.com/ClickHouse/ClickHouse/pull/60150) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#60279](https://github.com/ClickHouse/ClickHouse/issues/60279): Hide sensitive info for `S3Queue` table engine. [#60233](https://github.com/ClickHouse/ClickHouse/pull/60233) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#61000](https://github.com/ClickHouse/ClickHouse/issues/61000): Reduce the number of read rows from `system.numbers`. Fixes [#59418](https://github.com/ClickHouse/ClickHouse/issues/59418). [#60546](https://github.com/ClickHouse/ClickHouse/pull/60546) ([JackyWoo](https://github.com/JackyWoo)).
+* Backported in [#60791](https://github.com/ClickHouse/ClickHouse/issues/60791): Fix buffer overflow that can happen if the attacker asks the HTTP server to decompress data with a composition of codecs and size triggering numeric overflow. Fix buffer overflow that can happen inside codec NONE on wrong input data. This was submitted by TIANGONG research team through our [Bug Bounty program](https://github.com/ClickHouse/ClickHouse/issues/38986). [#60731](https://github.com/ClickHouse/ClickHouse/pull/60731) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#60783](https://github.com/ClickHouse/ClickHouse/issues/60783): Functions for SQL/JSON were able to read uninitialized memory. This closes [#60017](https://github.com/ClickHouse/ClickHouse/issues/60017). Found by Fuzzer. [#60738](https://github.com/ClickHouse/ClickHouse/pull/60738) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#60803](https://github.com/ClickHouse/ClickHouse/issues/60803): Do not set aws custom metadata `x-amz-meta-*` headers on UploadPart & CompleteMultipartUpload calls. [#60748](https://github.com/ClickHouse/ClickHouse/pull/60748) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)).
+* Backported in [#60820](https://github.com/ClickHouse/ClickHouse/issues/60820): Fix crash in arrayEnumerateRanked. [#60764](https://github.com/ClickHouse/ClickHouse/pull/60764) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#60841](https://github.com/ClickHouse/ClickHouse/issues/60841): Fix crash when using input() in INSERT SELECT JOIN. Closes [#60035](https://github.com/ClickHouse/ClickHouse/issues/60035). [#60765](https://github.com/ClickHouse/ClickHouse/pull/60765) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#60904](https://github.com/ClickHouse/ClickHouse/issues/60904): Avoid segfault if too many keys are skipped when reading from S3. [#60849](https://github.com/ClickHouse/ClickHouse/pull/60849) ([Antonio Andelic](https://github.com/antonio2368)).
+
+#### NO CL CATEGORY
+
+* Backported in [#60186](https://github.com/ClickHouse/ClickHouse/issues/60186):. [#60181](https://github.com/ClickHouse/ClickHouse/pull/60181) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#60333](https://github.com/ClickHouse/ClickHouse/issues/60333): CI: Fix job failures due to jepsen artifacts. [#59890](https://github.com/ClickHouse/ClickHouse/pull/59890) ([Max K.](https://github.com/maxknv)).
+* Backported in [#60034](https://github.com/ClickHouse/ClickHouse/issues/60034): Fix mark release ready. [#59994](https://github.com/ClickHouse/ClickHouse/pull/59994) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#60326](https://github.com/ClickHouse/ClickHouse/issues/60326): Ability to detect undead ZooKeeper sessions. [#60044](https://github.com/ClickHouse/ClickHouse/pull/60044) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#60363](https://github.com/ClickHouse/ClickHouse/issues/60363): CI: hot fix for gh statuses. [#60201](https://github.com/ClickHouse/ClickHouse/pull/60201) ([Max K.](https://github.com/maxknv)).
+* Backported in [#60648](https://github.com/ClickHouse/ClickHouse/issues/60648): Detect io_uring in tests. [#60373](https://github.com/ClickHouse/ClickHouse/pull/60373) ([Azat Khuzhin](https://github.com/azat)).
+* Backported in [#60569](https://github.com/ClickHouse/ClickHouse/issues/60569): Remove broken test while we fix it. [#60547](https://github.com/ClickHouse/ClickHouse/pull/60547) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#60756](https://github.com/ClickHouse/ClickHouse/issues/60756): Update shellcheck. [#60553](https://github.com/ClickHouse/ClickHouse/pull/60553) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#60584](https://github.com/ClickHouse/ClickHouse/issues/60584): CI: fix docker build job name. [#60554](https://github.com/ClickHouse/ClickHouse/pull/60554) ([Max K.](https://github.com/maxknv)).
+
diff --git a/docs/changelogs/v24.3.4.147-lts.md b/docs/changelogs/v24.3.4.147-lts.md
new file mode 100644
index 00000000000..7d77fb29977
--- /dev/null
+++ b/docs/changelogs/v24.3.4.147-lts.md
@@ -0,0 +1,100 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.3.4.147-lts (31a7bdc346d) FIXME as compared to v24.3.3.102-lts (7e7f3bdd9be)
+
+#### Improvement
+* Backported in [#63465](https://github.com/ClickHouse/ClickHouse/issues/63465): Make rabbitmq nack broken messages. Closes [#45350](https://github.com/ClickHouse/ClickHouse/issues/45350). [#60312](https://github.com/ClickHouse/ClickHouse/pull/60312) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#64290](https://github.com/ClickHouse/ClickHouse/issues/64290): Fix logical-error when undoing quorum insert transaction. [#61953](https://github.com/ClickHouse/ClickHouse/pull/61953) ([Han Fei](https://github.com/hanfei1991)).
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#63610](https://github.com/ClickHouse/ClickHouse/issues/63610): The Dockerfile is reviewed by the docker official library in https://github.com/docker-library/official-images/pull/15846. [#63400](https://github.com/ClickHouse/ClickHouse/pull/63400) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#65128](https://github.com/ClickHouse/ClickHouse/issues/65128): Decrease the `unit-test` image a few times. [#65102](https://github.com/ClickHouse/ClickHouse/pull/65102) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Backported in [#64277](https://github.com/ClickHouse/ClickHouse/issues/64277): Fix queries with FINAL give wrong result when table does not use adaptive granularity. [#62432](https://github.com/ClickHouse/ClickHouse/pull/62432) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#63716](https://github.com/ClickHouse/ClickHouse/issues/63716): Fix excessive memory usage for queries with nested lambdas. Fixes [#62036](https://github.com/ClickHouse/ClickHouse/issues/62036). [#62462](https://github.com/ClickHouse/ClickHouse/pull/62462) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#63247](https://github.com/ClickHouse/ClickHouse/issues/63247): Fix size checks when updating materialized nested columns ( fixes [#62731](https://github.com/ClickHouse/ClickHouse/issues/62731) ). [#62773](https://github.com/ClickHouse/ClickHouse/pull/62773) ([Eliot Hautefeuille](https://github.com/hileef)).
+* Backported in [#62984](https://github.com/ClickHouse/ClickHouse/issues/62984): Fix the `Unexpected return type` error for queries that read from `StorageBuffer` with `PREWHERE` when the source table has different types. Fixes [#62545](https://github.com/ClickHouse/ClickHouse/issues/62545). [#62916](https://github.com/ClickHouse/ClickHouse/pull/62916) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#63185](https://github.com/ClickHouse/ClickHouse/issues/63185): Sanity check: Clamp values instead of throwing. [#63119](https://github.com/ClickHouse/ClickHouse/pull/63119) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#63293](https://github.com/ClickHouse/ClickHouse/issues/63293): Fix crash with untuple and unresolved lambda. [#63131](https://github.com/ClickHouse/ClickHouse/pull/63131) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#63411](https://github.com/ClickHouse/ClickHouse/issues/63411): Fix a misbehavior when SQL security defaults don't load for old tables during server startup. [#63209](https://github.com/ClickHouse/ClickHouse/pull/63209) ([pufit](https://github.com/pufit)).
+* Backported in [#63616](https://github.com/ClickHouse/ClickHouse/issues/63616): Fix bug which could potentially lead to rare LOGICAL_ERROR during SELECT query with message: `Unexpected return type from materialize. Expected type_XXX. Got type_YYY.` Introduced in [#59379](https://github.com/ClickHouse/ClickHouse/issues/59379). [#63353](https://github.com/ClickHouse/ClickHouse/pull/63353) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63455](https://github.com/ClickHouse/ClickHouse/issues/63455): Fix `X-ClickHouse-Timezone` header returning wrong timezone when using `session_timezone` as query level setting. [#63377](https://github.com/ClickHouse/ClickHouse/pull/63377) ([Andrey Zvonov](https://github.com/zvonand)).
+* Backported in [#63603](https://github.com/ClickHouse/ClickHouse/issues/63603): Fix backup of projection part in case projection was removed from table metadata, but part still has projection. [#63426](https://github.com/ClickHouse/ClickHouse/pull/63426) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#63508](https://github.com/ClickHouse/ClickHouse/issues/63508): Fix 'Every derived table must have its own alias' error for MYSQL dictionary source, close [#63341](https://github.com/ClickHouse/ClickHouse/issues/63341). [#63481](https://github.com/ClickHouse/ClickHouse/pull/63481) ([vdimir](https://github.com/vdimir)).
+* Backported in [#63595](https://github.com/ClickHouse/ClickHouse/issues/63595): Avoid segafult in `MergeTreePrefetchedReadPool` while fetching projection parts. [#63513](https://github.com/ClickHouse/ClickHouse/pull/63513) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#63748](https://github.com/ClickHouse/ClickHouse/issues/63748): Read only the necessary columns from VIEW (new analyzer). Closes [#62594](https://github.com/ClickHouse/ClickHouse/issues/62594). [#63688](https://github.com/ClickHouse/ClickHouse/pull/63688) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#63770](https://github.com/ClickHouse/ClickHouse/issues/63770): Fix [#63539](https://github.com/ClickHouse/ClickHouse/issues/63539). Forbid WINDOW redefinition in new analyzer. [#63694](https://github.com/ClickHouse/ClickHouse/pull/63694) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#64189](https://github.com/ClickHouse/ClickHouse/issues/64189): Fix `Not found column` and `CAST AS Map from array requires nested tuple of 2 elements` exceptions for distributed queries which use `Map(Nothing, Nothing)` type. Fixes [#63637](https://github.com/ClickHouse/ClickHouse/issues/63637). [#63753](https://github.com/ClickHouse/ClickHouse/pull/63753) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#63845](https://github.com/ClickHouse/ClickHouse/issues/63845): Fix possible `ILLEGAL_COLUMN` error in `partial_merge` join, close [#37928](https://github.com/ClickHouse/ClickHouse/issues/37928). [#63755](https://github.com/ClickHouse/ClickHouse/pull/63755) ([vdimir](https://github.com/vdimir)).
+* Backported in [#63906](https://github.com/ClickHouse/ClickHouse/issues/63906): `query_plan_remove_redundant_distinct` can break queries with WINDOW FUNCTIONS (with `allow_experimental_analyzer` is on). Fixes [#62820](https://github.com/ClickHouse/ClickHouse/issues/62820). [#63776](https://github.com/ClickHouse/ClickHouse/pull/63776) ([Igor Nikonov](https://github.com/devcrafter)).
+* Backported in [#63989](https://github.com/ClickHouse/ClickHouse/issues/63989): Fix incorrect select query result when parallel replicas were used to read from a Materialized View. [#63861](https://github.com/ClickHouse/ClickHouse/pull/63861) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#64031](https://github.com/ClickHouse/ClickHouse/issues/64031): Fix a error `Database name is empty` for remote queries with lambdas over the cluster with modified default database. Fixes [#63471](https://github.com/ClickHouse/ClickHouse/issues/63471). [#63864](https://github.com/ClickHouse/ClickHouse/pull/63864) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#64559](https://github.com/ClickHouse/ClickHouse/issues/64559): Fix SIGSEGV due to CPU/Real (`query_profiler_real_time_period_ns`/`query_profiler_cpu_time_period_ns`) profiler (has been an issue since 2022, that leads to periodic server crashes, especially if you were using distributed engine). [#63865](https://github.com/ClickHouse/ClickHouse/pull/63865) ([Azat Khuzhin](https://github.com/azat)).
+* Backported in [#64009](https://github.com/ClickHouse/ClickHouse/issues/64009): Fix analyzer - IN function with arbitrary deep sub-selects in materialized view to use insertion block. [#63930](https://github.com/ClickHouse/ClickHouse/pull/63930) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Backported in [#64236](https://github.com/ClickHouse/ClickHouse/issues/64236): Fix resolve of unqualified COLUMNS matcher. Preserve the input columns order and forbid usage of unknown identifiers. [#63962](https://github.com/ClickHouse/ClickHouse/pull/63962) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#64106](https://github.com/ClickHouse/ClickHouse/issues/64106): Deserialize untrusted binary inputs in a safer way. [#64024](https://github.com/ClickHouse/ClickHouse/pull/64024) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#64168](https://github.com/ClickHouse/ClickHouse/issues/64168): Add missing settings to recoverLostReplica. [#64040](https://github.com/ClickHouse/ClickHouse/pull/64040) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#64320](https://github.com/ClickHouse/ClickHouse/issues/64320): This fix will use a proper redefined context with the correct definer for each individual view in the query pipeline Closes [#63777](https://github.com/ClickHouse/ClickHouse/issues/63777). [#64079](https://github.com/ClickHouse/ClickHouse/pull/64079) ([pufit](https://github.com/pufit)).
+* Backported in [#64380](https://github.com/ClickHouse/ClickHouse/issues/64380): Fix analyzer: "Not found column" error is fixed when using INTERPOLATE. [#64096](https://github.com/ClickHouse/ClickHouse/pull/64096) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Backported in [#64567](https://github.com/ClickHouse/ClickHouse/issues/64567): Fix creating backups to S3 buckets with different credentials from the disk containing the file. [#64153](https://github.com/ClickHouse/ClickHouse/pull/64153) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#64270](https://github.com/ClickHouse/ClickHouse/issues/64270): Prevent LOGICAL_ERROR on CREATE TABLE as MaterializedView. [#64174](https://github.com/ClickHouse/ClickHouse/pull/64174) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#64339](https://github.com/ClickHouse/ClickHouse/issues/64339): The query cache now considers two identical queries against different databases as different. The previous behavior could be used to bypass missing privileges to read from a table. [#64199](https://github.com/ClickHouse/ClickHouse/pull/64199) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#64259](https://github.com/ClickHouse/ClickHouse/issues/64259): Ignore `text_log` config when using Keeper. [#64218](https://github.com/ClickHouse/ClickHouse/pull/64218) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#64688](https://github.com/ClickHouse/ClickHouse/issues/64688): Fix Query Tree size validation. Closes [#63701](https://github.com/ClickHouse/ClickHouse/issues/63701). [#64377](https://github.com/ClickHouse/ClickHouse/pull/64377) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#64725](https://github.com/ClickHouse/ClickHouse/issues/64725): Fixed `CREATE TABLE AS` queries for tables with default expressions. [#64455](https://github.com/ClickHouse/ClickHouse/pull/64455) ([Anton Popov](https://github.com/CurtizJ)).
+* Backported in [#64621](https://github.com/ClickHouse/ClickHouse/issues/64621): Fix an error `Cannot find column` in distributed queries with constant CTE in the `GROUP BY` key. [#64519](https://github.com/ClickHouse/ClickHouse/pull/64519) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#64678](https://github.com/ClickHouse/ClickHouse/issues/64678): Fix [#64612](https://github.com/ClickHouse/ClickHouse/issues/64612). Do not rewrite aggregation if `-If` combinator is already used. [#64638](https://github.com/ClickHouse/ClickHouse/pull/64638) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#64831](https://github.com/ClickHouse/ClickHouse/issues/64831): Fix bug which could lead to non-working TTLs with expressions. Fixes [#63700](https://github.com/ClickHouse/ClickHouse/issues/63700). [#64694](https://github.com/ClickHouse/ClickHouse/pull/64694) ([alesapin](https://github.com/alesapin)).
+* Backported in [#64940](https://github.com/ClickHouse/ClickHouse/issues/64940): Fix OrderByLimitByDuplicateEliminationVisitor across subqueries. [#64766](https://github.com/ClickHouse/ClickHouse/pull/64766) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#64869](https://github.com/ClickHouse/ClickHouse/issues/64869): Fixed memory possible incorrect memory tracking in several kinds of queries: queries that read any data from S3, queries via http protocol, asynchronous inserts. [#64844](https://github.com/ClickHouse/ClickHouse/pull/64844) ([Anton Popov](https://github.com/CurtizJ)).
+* Backported in [#64980](https://github.com/ClickHouse/ClickHouse/issues/64980): Fix the `Block structure mismatch` error for queries reading with `PREWHERE` from the materialized view when the materialized view has columns of different types than the source table. Fixes [#64611](https://github.com/ClickHouse/ClickHouse/issues/64611). [#64855](https://github.com/ClickHouse/ClickHouse/pull/64855) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#64972](https://github.com/ClickHouse/ClickHouse/issues/64972): Fix rare crash when table has TTL with subquery + database replicated + parallel replicas + analyzer. It's really rare, but please don't use TTLs with subqueries. [#64858](https://github.com/ClickHouse/ClickHouse/pull/64858) ([alesapin](https://github.com/alesapin)).
+* Backported in [#65070](https://github.com/ClickHouse/ClickHouse/issues/65070): Fix `ALTER MODIFY COMMENT` query that was broken for parameterized VIEWs in https://github.com/ClickHouse/ClickHouse/pull/54211. [#65031](https://github.com/ClickHouse/ClickHouse/pull/65031) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Backported in [#65175](https://github.com/ClickHouse/ClickHouse/issues/65175): Fix the `Unknown expression identifier` error for remote queries with `INTERPOLATE (alias)` (new analyzer). Fixes [#64636](https://github.com/ClickHouse/ClickHouse/issues/64636). [#65090](https://github.com/ClickHouse/ClickHouse/pull/65090) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+
+* Backported in [#64587](https://github.com/ClickHouse/ClickHouse/issues/64587): Disabled `enable_vertical_final` setting by default. This feature should not be used because it has a bug: [#64543](https://github.com/ClickHouse/ClickHouse/issues/64543). [#64544](https://github.com/ClickHouse/ClickHouse/pull/64544) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#64878](https://github.com/ClickHouse/ClickHouse/issues/64878): This PR fixes an error when a user in a specific situation can escalate their privileges on the default database without necessary grants. [#64769](https://github.com/ClickHouse/ClickHouse/pull/64769) ([pufit](https://github.com/pufit)).
+
+#### NO CL CATEGORY
+
+* Backported in [#63304](https://github.com/ClickHouse/ClickHouse/issues/63304):. [#63297](https://github.com/ClickHouse/ClickHouse/pull/63297) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#63708](https://github.com/ClickHouse/ClickHouse/issues/63708):. [#63415](https://github.com/ClickHouse/ClickHouse/pull/63415) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+
+#### NO CL ENTRY
+
+* NO CL ENTRY: 'Revert "Backport [#64363](https://github.com/ClickHouse/ClickHouse/issues/64363) to 24.3: Split tests 03039_dynamic_all_merge_algorithms to avoid timeouts"'. [#64907](https://github.com/ClickHouse/ClickHouse/pull/64907) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#63751](https://github.com/ClickHouse/ClickHouse/issues/63751): group_by_use_nulls strikes back. [#62922](https://github.com/ClickHouse/ClickHouse/pull/62922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#63558](https://github.com/ClickHouse/ClickHouse/issues/63558): Try fix segfault in `MergeTreeReadPoolBase::createTask`. [#63323](https://github.com/ClickHouse/ClickHouse/pull/63323) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#63336](https://github.com/ClickHouse/ClickHouse/issues/63336): The commit url has different pattern. [#63331](https://github.com/ClickHouse/ClickHouse/pull/63331) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#63374](https://github.com/ClickHouse/ClickHouse/issues/63374): Add tags for the test 03000_traverse_shadow_system_data_paths.sql to make it stable. [#63366](https://github.com/ClickHouse/ClickHouse/pull/63366) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Backported in [#63625](https://github.com/ClickHouse/ClickHouse/issues/63625): Workaround for `oklch()` inside canvas bug for firefox. [#63404](https://github.com/ClickHouse/ClickHouse/pull/63404) ([Sergei Trifonov](https://github.com/serxa)).
+* Backported in [#63569](https://github.com/ClickHouse/ClickHouse/issues/63569): Add `jwcrypto` to integration tests runner. [#63551](https://github.com/ClickHouse/ClickHouse/pull/63551) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Backported in [#63649](https://github.com/ClickHouse/ClickHouse/issues/63649): Fix `02362_part_log_merge_algorithm` flaky test. [#63635](https://github.com/ClickHouse/ClickHouse/pull/63635) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
+* Backported in [#63762](https://github.com/ClickHouse/ClickHouse/issues/63762): Cancel S3 reads properly when parallel reads are used. [#63687](https://github.com/ClickHouse/ClickHouse/pull/63687) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#63741](https://github.com/ClickHouse/ClickHouse/issues/63741): Userspace page cache: don't collect stats if cache is unused. [#63730](https://github.com/ClickHouse/ClickHouse/pull/63730) ([Michael Kolupaev](https://github.com/al13n321)).
+* Backported in [#63826](https://github.com/ClickHouse/ClickHouse/issues/63826): Fix `test_odbc_interaction` for arm64 on linux. [#63787](https://github.com/ClickHouse/ClickHouse/pull/63787) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63895](https://github.com/ClickHouse/ClickHouse/issues/63895): Fix `test_catboost_evaluate` for aarch64. [#63789](https://github.com/ClickHouse/ClickHouse/pull/63789) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63887](https://github.com/ClickHouse/ClickHouse/issues/63887): Fix `test_disk_types` for aarch64. [#63832](https://github.com/ClickHouse/ClickHouse/pull/63832) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63879](https://github.com/ClickHouse/ClickHouse/issues/63879): Fix `test_short_strings_aggregation` for arm. [#63836](https://github.com/ClickHouse/ClickHouse/pull/63836) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63916](https://github.com/ClickHouse/ClickHouse/issues/63916): Disable `test_non_default_compression/test.py::test_preconfigured_deflateqpl_codec` on arm. [#63839](https://github.com/ClickHouse/ClickHouse/pull/63839) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63969](https://github.com/ClickHouse/ClickHouse/issues/63969): fix 02124_insert_deduplication_token_multiple_blocks. [#63950](https://github.com/ClickHouse/ClickHouse/pull/63950) ([Han Fei](https://github.com/hanfei1991)).
+* Backported in [#64047](https://github.com/ClickHouse/ClickHouse/issues/64047): Do not create new release in release branch automatically. [#64039](https://github.com/ClickHouse/ClickHouse/pull/64039) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#64076](https://github.com/ClickHouse/ClickHouse/issues/64076): Files without shebang have mime 'text/plain' or 'inode/x-empty'. [#64062](https://github.com/ClickHouse/ClickHouse/pull/64062) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#64142](https://github.com/ClickHouse/ClickHouse/issues/64142): Fix sanitizers. [#64090](https://github.com/ClickHouse/ClickHouse/pull/64090) ([Azat Khuzhin](https://github.com/azat)).
+* Backported in [#64159](https://github.com/ClickHouse/ClickHouse/issues/64159): Add retries in `git submodule update`. [#64125](https://github.com/ClickHouse/ClickHouse/pull/64125) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#64473](https://github.com/ClickHouse/ClickHouse/issues/64473): Split tests 03039_dynamic_all_merge_algorithms to avoid timeouts. [#64363](https://github.com/ClickHouse/ClickHouse/pull/64363) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#65113](https://github.com/ClickHouse/ClickHouse/issues/65113): Adjust the `version_helper` and script to a new release scheme. [#64759](https://github.com/ClickHouse/ClickHouse/pull/64759) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#64999](https://github.com/ClickHouse/ClickHouse/issues/64999): Fix crash with DISTINCT and window functions. [#64767](https://github.com/ClickHouse/ClickHouse/pull/64767) ([Igor Nikonov](https://github.com/devcrafter)).
+
diff --git a/docs/changelogs/v24.4.2.141-stable.md b/docs/changelogs/v24.4.2.141-stable.md
new file mode 100644
index 00000000000..656d0854392
--- /dev/null
+++ b/docs/changelogs/v24.4.2.141-stable.md
@@ -0,0 +1,101 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.4.2.141-stable (9e23d27bd11) FIXME as compared to v24.4.1.2088-stable (6d4b31322d1)
+
+#### Improvement
+* Backported in [#63467](https://github.com/ClickHouse/ClickHouse/issues/63467): Make rabbitmq nack broken messages. Closes [#45350](https://github.com/ClickHouse/ClickHouse/issues/45350). [#60312](https://github.com/ClickHouse/ClickHouse/pull/60312) ([Kseniia Sumarokova](https://github.com/kssenii)).
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#63612](https://github.com/ClickHouse/ClickHouse/issues/63612): The Dockerfile is reviewed by the docker official library in https://github.com/docker-library/official-images/pull/15846. [#63400](https://github.com/ClickHouse/ClickHouse/pull/63400) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Backported in [#64279](https://github.com/ClickHouse/ClickHouse/issues/64279): Fix queries with FINAL give wrong result when table does not use adaptive granularity. [#62432](https://github.com/ClickHouse/ClickHouse/pull/62432) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#63295](https://github.com/ClickHouse/ClickHouse/issues/63295): Fix crash with untuple and unresolved lambda. [#63131](https://github.com/ClickHouse/ClickHouse/pull/63131) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#63978](https://github.com/ClickHouse/ClickHouse/issues/63978): Fix intersect parts when restart after drop range. [#63202](https://github.com/ClickHouse/ClickHouse/pull/63202) ([Han Fei](https://github.com/hanfei1991)).
+* Backported in [#63413](https://github.com/ClickHouse/ClickHouse/issues/63413): Fix a misbehavior when SQL security defaults don't load for old tables during server startup. [#63209](https://github.com/ClickHouse/ClickHouse/pull/63209) ([pufit](https://github.com/pufit)).
+* Backported in [#63388](https://github.com/ClickHouse/ClickHouse/issues/63388): JOIN filter push down filled join fix. Closes [#63228](https://github.com/ClickHouse/ClickHouse/issues/63228). [#63234](https://github.com/ClickHouse/ClickHouse/pull/63234) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#63618](https://github.com/ClickHouse/ClickHouse/issues/63618): Fix bug which could potentially lead to rare LOGICAL_ERROR during SELECT query with message: `Unexpected return type from materialize. Expected type_XXX. Got type_YYY.` Introduced in [#59379](https://github.com/ClickHouse/ClickHouse/issues/59379). [#63353](https://github.com/ClickHouse/ClickHouse/pull/63353) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63451](https://github.com/ClickHouse/ClickHouse/issues/63451): Fix `X-ClickHouse-Timezone` header returning wrong timezone when using `session_timezone` as query level setting. [#63377](https://github.com/ClickHouse/ClickHouse/pull/63377) ([Andrey Zvonov](https://github.com/zvonand)).
+* Backported in [#63605](https://github.com/ClickHouse/ClickHouse/issues/63605): Fix backup of projection part in case projection was removed from table metadata, but part still has projection. [#63426](https://github.com/ClickHouse/ClickHouse/pull/63426) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#63510](https://github.com/ClickHouse/ClickHouse/issues/63510): Fix 'Every derived table must have its own alias' error for MYSQL dictionary source, close [#63341](https://github.com/ClickHouse/ClickHouse/issues/63341). [#63481](https://github.com/ClickHouse/ClickHouse/pull/63481) ([vdimir](https://github.com/vdimir)).
+* Backported in [#63592](https://github.com/ClickHouse/ClickHouse/issues/63592): Avoid segafult in `MergeTreePrefetchedReadPool` while fetching projection parts. [#63513](https://github.com/ClickHouse/ClickHouse/pull/63513) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#63750](https://github.com/ClickHouse/ClickHouse/issues/63750): Read only the necessary columns from VIEW (new analyzer). Closes [#62594](https://github.com/ClickHouse/ClickHouse/issues/62594). [#63688](https://github.com/ClickHouse/ClickHouse/pull/63688) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#63772](https://github.com/ClickHouse/ClickHouse/issues/63772): Fix [#63539](https://github.com/ClickHouse/ClickHouse/issues/63539). Forbid WINDOW redefinition in new analyzer. [#63694](https://github.com/ClickHouse/ClickHouse/pull/63694) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#63872](https://github.com/ClickHouse/ClickHouse/issues/63872): Flatten_nested is broken with replicated database. [#63695](https://github.com/ClickHouse/ClickHouse/pull/63695) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#63854](https://github.com/ClickHouse/ClickHouse/issues/63854): Fix `Not found column` and `CAST AS Map from array requires nested tuple of 2 elements` exceptions for distributed queries which use `Map(Nothing, Nothing)` type. Fixes [#63637](https://github.com/ClickHouse/ClickHouse/issues/63637). [#63753](https://github.com/ClickHouse/ClickHouse/pull/63753) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#63847](https://github.com/ClickHouse/ClickHouse/issues/63847): Fix possible `ILLEGAL_COLUMN` error in `partial_merge` join, close [#37928](https://github.com/ClickHouse/ClickHouse/issues/37928). [#63755](https://github.com/ClickHouse/ClickHouse/pull/63755) ([vdimir](https://github.com/vdimir)).
+* Backported in [#63908](https://github.com/ClickHouse/ClickHouse/issues/63908): `query_plan_remove_redundant_distinct` can break queries with WINDOW FUNCTIONS (with `allow_experimental_analyzer` is on). Fixes [#62820](https://github.com/ClickHouse/ClickHouse/issues/62820). [#63776](https://github.com/ClickHouse/ClickHouse/pull/63776) ([Igor Nikonov](https://github.com/devcrafter)).
+* Backported in [#63955](https://github.com/ClickHouse/ClickHouse/issues/63955): Fix possible crash with SYSTEM UNLOAD PRIMARY KEY. [#63778](https://github.com/ClickHouse/ClickHouse/pull/63778) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#63938](https://github.com/ClickHouse/ClickHouse/issues/63938): Allow JOIN filter push down to both streams if only single equivalent column is used in query. Closes [#63799](https://github.com/ClickHouse/ClickHouse/issues/63799). [#63819](https://github.com/ClickHouse/ClickHouse/pull/63819) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#63991](https://github.com/ClickHouse/ClickHouse/issues/63991): Fix incorrect select query result when parallel replicas were used to read from a Materialized View. [#63861](https://github.com/ClickHouse/ClickHouse/pull/63861) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#64033](https://github.com/ClickHouse/ClickHouse/issues/64033): Fix a error `Database name is empty` for remote queries with lambdas over the cluster with modified default database. Fixes [#63471](https://github.com/ClickHouse/ClickHouse/issues/63471). [#63864](https://github.com/ClickHouse/ClickHouse/pull/63864) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#64561](https://github.com/ClickHouse/ClickHouse/issues/64561): Fix SIGSEGV due to CPU/Real (`query_profiler_real_time_period_ns`/`query_profiler_cpu_time_period_ns`) profiler (has been an issue since 2022, that leads to periodic server crashes, especially if you were using distributed engine). [#63865](https://github.com/ClickHouse/ClickHouse/pull/63865) ([Azat Khuzhin](https://github.com/azat)).
+* Backported in [#64011](https://github.com/ClickHouse/ClickHouse/issues/64011): Fix analyzer - IN function with arbitrary deep sub-selects in materialized view to use insertion block. [#63930](https://github.com/ClickHouse/ClickHouse/pull/63930) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Backported in [#64238](https://github.com/ClickHouse/ClickHouse/issues/64238): Fix resolve of unqualified COLUMNS matcher. Preserve the input columns order and forbid usage of unknown identifiers. [#63962](https://github.com/ClickHouse/ClickHouse/pull/63962) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#64103](https://github.com/ClickHouse/ClickHouse/issues/64103): Deserialize untrusted binary inputs in a safer way. [#64024](https://github.com/ClickHouse/ClickHouse/pull/64024) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#64170](https://github.com/ClickHouse/ClickHouse/issues/64170): Add missing settings to recoverLostReplica. [#64040](https://github.com/ClickHouse/ClickHouse/pull/64040) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#64322](https://github.com/ClickHouse/ClickHouse/issues/64322): This fix will use a proper redefined context with the correct definer for each individual view in the query pipeline Closes [#63777](https://github.com/ClickHouse/ClickHouse/issues/63777). [#64079](https://github.com/ClickHouse/ClickHouse/pull/64079) ([pufit](https://github.com/pufit)).
+* Backported in [#64382](https://github.com/ClickHouse/ClickHouse/issues/64382): Fix analyzer: "Not found column" error is fixed when using INTERPOLATE. [#64096](https://github.com/ClickHouse/ClickHouse/pull/64096) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Backported in [#64568](https://github.com/ClickHouse/ClickHouse/issues/64568): Fix creating backups to S3 buckets with different credentials from the disk containing the file. [#64153](https://github.com/ClickHouse/ClickHouse/pull/64153) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#64272](https://github.com/ClickHouse/ClickHouse/issues/64272): Prevent LOGICAL_ERROR on CREATE TABLE as MaterializedView. [#64174](https://github.com/ClickHouse/ClickHouse/pull/64174) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#64330](https://github.com/ClickHouse/ClickHouse/issues/64330): The query cache now considers two identical queries against different databases as different. The previous behavior could be used to bypass missing privileges to read from a table. [#64199](https://github.com/ClickHouse/ClickHouse/pull/64199) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#64254](https://github.com/ClickHouse/ClickHouse/issues/64254): Ignore `text_log` config when using Keeper. [#64218](https://github.com/ClickHouse/ClickHouse/pull/64218) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#64690](https://github.com/ClickHouse/ClickHouse/issues/64690): Fix Query Tree size validation. Closes [#63701](https://github.com/ClickHouse/ClickHouse/issues/63701). [#64377](https://github.com/ClickHouse/ClickHouse/pull/64377) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#64409](https://github.com/ClickHouse/ClickHouse/issues/64409): Fix `Logical error: Bad cast` for `Buffer` table with `PREWHERE`. Fixes [#64172](https://github.com/ClickHouse/ClickHouse/issues/64172). [#64388](https://github.com/ClickHouse/ClickHouse/pull/64388) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#64727](https://github.com/ClickHouse/ClickHouse/issues/64727): Fixed `CREATE TABLE AS` queries for tables with default expressions. [#64455](https://github.com/ClickHouse/ClickHouse/pull/64455) ([Anton Popov](https://github.com/CurtizJ)).
+* Backported in [#64623](https://github.com/ClickHouse/ClickHouse/issues/64623): Fix an error `Cannot find column` in distributed queries with constant CTE in the `GROUP BY` key. [#64519](https://github.com/ClickHouse/ClickHouse/pull/64519) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#64680](https://github.com/ClickHouse/ClickHouse/issues/64680): Fix [#64612](https://github.com/ClickHouse/ClickHouse/issues/64612). Do not rewrite aggregation if `-If` combinator is already used. [#64638](https://github.com/ClickHouse/ClickHouse/pull/64638) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#64942](https://github.com/ClickHouse/ClickHouse/issues/64942): Fix OrderByLimitByDuplicateEliminationVisitor across subqueries. [#64766](https://github.com/ClickHouse/ClickHouse/pull/64766) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#64871](https://github.com/ClickHouse/ClickHouse/issues/64871): Fixed memory possible incorrect memory tracking in several kinds of queries: queries that read any data from S3, queries via http protocol, asynchronous inserts. [#64844](https://github.com/ClickHouse/ClickHouse/pull/64844) ([Anton Popov](https://github.com/CurtizJ)).
+
+#### CI Fix or Improvement (changelog entry is not required)
+
+* Backported in [#63364](https://github.com/ClickHouse/ClickHouse/issues/63364): Implement cumulative A Sync status. [#61464](https://github.com/ClickHouse/ClickHouse/pull/61464) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#63338](https://github.com/ClickHouse/ClickHouse/issues/63338): Use `/commit/` to have the URLs in [reports](https://play.clickhouse.com/play?user=play#c2VsZWN0IGRpc3RpbmN0IGNvbW1pdF91cmwgZnJvbSBjaGVja3Mgd2hlcmUgY2hlY2tfc3RhcnRfdGltZSA+PSBub3coKSAtIGludGVydmFsIDEgbW9udGggYW5kIHB1bGxfcmVxdWVzdF9udW1iZXI9NjA1MzI=) like https://github.com/ClickHouse/ClickHouse/commit/44f8bc5308b53797bec8cccc3bd29fab8a00235d and not like https://github.com/ClickHouse/ClickHouse/commits/44f8bc5308b53797bec8cccc3bd29fab8a00235d. [#63331](https://github.com/ClickHouse/ClickHouse/pull/63331) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#63376](https://github.com/ClickHouse/ClickHouse/issues/63376):. [#63366](https://github.com/ClickHouse/ClickHouse/pull/63366) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Backported in [#63571](https://github.com/ClickHouse/ClickHouse/issues/63571):. [#63551](https://github.com/ClickHouse/ClickHouse/pull/63551) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Backported in [#63651](https://github.com/ClickHouse/ClickHouse/issues/63651): Fix 02362_part_log_merge_algorithm flaky test. [#63635](https://github.com/ClickHouse/ClickHouse/pull/63635) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
+* Backported in [#63828](https://github.com/ClickHouse/ClickHouse/issues/63828): Fix test_odbc_interaction from aarch64 [#61457](https://github.com/ClickHouse/ClickHouse/issues/61457). [#63787](https://github.com/ClickHouse/ClickHouse/pull/63787) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63897](https://github.com/ClickHouse/ClickHouse/issues/63897): Fix test `test_catboost_evaluate` for aarch64. [#61457](https://github.com/ClickHouse/ClickHouse/issues/61457). [#63789](https://github.com/ClickHouse/ClickHouse/pull/63789) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63889](https://github.com/ClickHouse/ClickHouse/issues/63889): Remove HDFS from disks config for one integration test for arm. [#61457](https://github.com/ClickHouse/ClickHouse/issues/61457). [#63832](https://github.com/ClickHouse/ClickHouse/pull/63832) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63881](https://github.com/ClickHouse/ClickHouse/issues/63881): Bump version for old image in test_short_strings_aggregation to make it work on arm. [#61457](https://github.com/ClickHouse/ClickHouse/issues/61457). [#63836](https://github.com/ClickHouse/ClickHouse/pull/63836) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63919](https://github.com/ClickHouse/ClickHouse/issues/63919): Disable test `test_non_default_compression/test.py::test_preconfigured_deflateqpl_codec` on arm. [#61457](https://github.com/ClickHouse/ClickHouse/issues/61457). [#63839](https://github.com/ClickHouse/ClickHouse/pull/63839) ([alesapin](https://github.com/alesapin)).
+* Backported in [#63971](https://github.com/ClickHouse/ClickHouse/issues/63971): Fix 02124_insert_deduplication_token_multiple_blocks. [#63950](https://github.com/ClickHouse/ClickHouse/pull/63950) ([Han Fei](https://github.com/hanfei1991)).
+* Backported in [#64049](https://github.com/ClickHouse/ClickHouse/issues/64049): Add `ClickHouseVersion.copy` method. Create a branch release in advance without spinning out the release to increase the stability. [#64039](https://github.com/ClickHouse/ClickHouse/pull/64039) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#64078](https://github.com/ClickHouse/ClickHouse/issues/64078): The mime type is not 100% reliable for Python and shell scripts without shebangs; add a check for file extension. [#64062](https://github.com/ClickHouse/ClickHouse/pull/64062) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#64161](https://github.com/ClickHouse/ClickHouse/issues/64161): Add retries in git submodule update. [#64125](https://github.com/ClickHouse/ClickHouse/pull/64125) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+
+* Backported in [#64589](https://github.com/ClickHouse/ClickHouse/issues/64589): Disabled `enable_vertical_final` setting by default. This feature should not be used because it has a bug: [#64543](https://github.com/ClickHouse/ClickHouse/issues/64543). [#64544](https://github.com/ClickHouse/ClickHouse/pull/64544) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#64880](https://github.com/ClickHouse/ClickHouse/issues/64880): This PR fixes an error when a user in a specific situation can escalate their privileges on the default database without necessary grants. [#64769](https://github.com/ClickHouse/ClickHouse/pull/64769) ([pufit](https://github.com/pufit)).
+
+#### NO CL CATEGORY
+
+* Backported in [#63306](https://github.com/ClickHouse/ClickHouse/issues/63306):. [#63297](https://github.com/ClickHouse/ClickHouse/pull/63297) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#63710](https://github.com/ClickHouse/ClickHouse/issues/63710):. [#63415](https://github.com/ClickHouse/ClickHouse/pull/63415) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+
+#### NO CL ENTRY
+
+* NO CL ENTRY: 'Revert "Backport [#64363](https://github.com/ClickHouse/ClickHouse/issues/64363) to 24.4: Split tests 03039_dynamic_all_merge_algorithms to avoid timeouts"'. [#64905](https://github.com/ClickHouse/ClickHouse/pull/64905) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* group_by_use_nulls strikes back [#62922](https://github.com/ClickHouse/ClickHouse/pull/62922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Add `FROM` keyword to `TRUNCATE ALL TABLES` [#63241](https://github.com/ClickHouse/ClickHouse/pull/63241) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* More checks for concurrently deleted files and dirs in system.remote_data_paths [#63274](https://github.com/ClickHouse/ClickHouse/pull/63274) ([Alexander Gololobov](https://github.com/davenger)).
+* Try fix segfault in `MergeTreeReadPoolBase::createTask` [#63323](https://github.com/ClickHouse/ClickHouse/pull/63323) ([Antonio Andelic](https://github.com/antonio2368)).
+* Skip unaccessible table dirs in system.remote_data_paths [#63330](https://github.com/ClickHouse/ClickHouse/pull/63330) ([Alexander Gololobov](https://github.com/davenger)).
+* Workaround for `oklch()` inside canvas bug for firefox [#63404](https://github.com/ClickHouse/ClickHouse/pull/63404) ([Sergei Trifonov](https://github.com/serxa)).
+* Cancel S3 reads properly when parallel reads are used [#63687](https://github.com/ClickHouse/ClickHouse/pull/63687) ([Antonio Andelic](https://github.com/antonio2368)).
+* Userspace page cache: don't collect stats if cache is unused [#63730](https://github.com/ClickHouse/ClickHouse/pull/63730) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix sanitizers [#64090](https://github.com/ClickHouse/ClickHouse/pull/64090) ([Azat Khuzhin](https://github.com/azat)).
+* Split tests 03039_dynamic_all_merge_algorithms to avoid timeouts [#64363](https://github.com/ClickHouse/ClickHouse/pull/64363) ([Kruglov Pavel](https://github.com/Avogar)).
+* CI: Critical bugfix category in PR template [#64480](https://github.com/ClickHouse/ClickHouse/pull/64480) ([Max K.](https://github.com/maxknv)).
+
diff --git a/docs/changelogs/v24.4.3.25-stable.md b/docs/changelogs/v24.4.3.25-stable.md
new file mode 100644
index 00000000000..9582753c731
--- /dev/null
+++ b/docs/changelogs/v24.4.3.25-stable.md
@@ -0,0 +1,30 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.4.3.25-stable (a915dd4eda4) FIXME as compared to v24.4.2.141-stable (9e23d27bd11)
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#65130](https://github.com/ClickHouse/ClickHouse/issues/65130): Decrease the `unit-test` image a few times. [#65102](https://github.com/ClickHouse/ClickHouse/pull/65102) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Backported in [#64982](https://github.com/ClickHouse/ClickHouse/issues/64982): Fix the `Block structure mismatch` error for queries reading with `PREWHERE` from the materialized view when the materialized view has columns of different types than the source table. Fixes [#64611](https://github.com/ClickHouse/ClickHouse/issues/64611). [#64855](https://github.com/ClickHouse/ClickHouse/pull/64855) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#64974](https://github.com/ClickHouse/ClickHouse/issues/64974): Fix rare crash when table has TTL with subquery + database replicated + parallel replicas + analyzer. It's really rare, but please don't use TTLs with subqueries. [#64858](https://github.com/ClickHouse/ClickHouse/pull/64858) ([alesapin](https://github.com/alesapin)).
+* Backported in [#65072](https://github.com/ClickHouse/ClickHouse/issues/65072): Fix `ALTER MODIFY COMMENT` query that was broken for parameterized VIEWs in https://github.com/ClickHouse/ClickHouse/pull/54211. [#65031](https://github.com/ClickHouse/ClickHouse/pull/65031) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Backported in [#65177](https://github.com/ClickHouse/ClickHouse/issues/65177): Fix the `Unknown expression identifier` error for remote queries with `INTERPOLATE (alias)` (new analyzer). Fixes [#64636](https://github.com/ClickHouse/ClickHouse/issues/64636). [#65090](https://github.com/ClickHouse/ClickHouse/pull/65090) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65263](https://github.com/ClickHouse/ClickHouse/issues/65263): Fix the bug in Hashed and Hashed_Array dictionary short circuit evaluation, which may read uninitialized number, leading to various errors. [#65256](https://github.com/ClickHouse/ClickHouse/pull/65256) ([jsc0218](https://github.com/jsc0218)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+
+* Backported in [#65285](https://github.com/ClickHouse/ClickHouse/issues/65285): Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#65114](https://github.com/ClickHouse/ClickHouse/issues/65114): Adjust the `version_helper` and script to a new release scheme. [#64759](https://github.com/ClickHouse/ClickHouse/pull/64759) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#65225](https://github.com/ClickHouse/ClickHouse/issues/65225): Capture weak_ptr of ContextAccess for safety. [#65051](https://github.com/ClickHouse/ClickHouse/pull/65051) ([Alexander Gololobov](https://github.com/davenger)).
+* Backported in [#65217](https://github.com/ClickHouse/ClickHouse/issues/65217): Fix false positives leaky memory warnings in OpenSSL. [#65125](https://github.com/ClickHouse/ClickHouse/pull/65125) ([Robert Schulze](https://github.com/rschu1ze)).
+
diff --git a/docs/changelogs/v24.5.2.34-stable.md b/docs/changelogs/v24.5.2.34-stable.md
new file mode 100644
index 00000000000..2db05a5f5dc
--- /dev/null
+++ b/docs/changelogs/v24.5.2.34-stable.md
@@ -0,0 +1,38 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.5.2.34-stable (45589aeee49) FIXME as compared to v24.5.1.1763-stable (647c154a94d)
+
+#### Improvement
+* Backported in [#65096](https://github.com/ClickHouse/ClickHouse/issues/65096): The setting `allow_experimental_join_condition` was accidentally marked as important which may prevent distributed queries in a mixed versions cluster from being executed successfully. [#65008](https://github.com/ClickHouse/ClickHouse/pull/65008) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#65132](https://github.com/ClickHouse/ClickHouse/issues/65132): Decrease the `unit-test` image a few times. [#65102](https://github.com/ClickHouse/ClickHouse/pull/65102) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Backported in [#64729](https://github.com/ClickHouse/ClickHouse/issues/64729): Fixed `CREATE TABLE AS` queries for tables with default expressions. [#64455](https://github.com/ClickHouse/ClickHouse/pull/64455) ([Anton Popov](https://github.com/CurtizJ)).
+* Backported in [#65061](https://github.com/ClickHouse/ClickHouse/issues/65061): Fix the `Expression nodes list expected 1 projection names` and `Unknown expression or identifier` errors for queries with aliases to `GLOBAL IN.` Fixes [#64445](https://github.com/ClickHouse/ClickHouse/issues/64445). [#64517](https://github.com/ClickHouse/ClickHouse/pull/64517) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65088](https://github.com/ClickHouse/ClickHouse/issues/65088): Fix removing the `WHERE` and `PREWHERE` expressions, which are always true (for the new analyzer). Fixes [#64575](https://github.com/ClickHouse/ClickHouse/issues/64575). [#64695](https://github.com/ClickHouse/ClickHouse/pull/64695) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#64944](https://github.com/ClickHouse/ClickHouse/issues/64944): Fix OrderByLimitByDuplicateEliminationVisitor across subqueries. [#64766](https://github.com/ClickHouse/ClickHouse/pull/64766) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#64873](https://github.com/ClickHouse/ClickHouse/issues/64873): Fixed memory possible incorrect memory tracking in several kinds of queries: queries that read any data from S3, queries via http protocol, asynchronous inserts. [#64844](https://github.com/ClickHouse/ClickHouse/pull/64844) ([Anton Popov](https://github.com/CurtizJ)).
+* Backported in [#64984](https://github.com/ClickHouse/ClickHouse/issues/64984): Fix the `Block structure mismatch` error for queries reading with `PREWHERE` from the materialized view when the materialized view has columns of different types than the source table. Fixes [#64611](https://github.com/ClickHouse/ClickHouse/issues/64611). [#64855](https://github.com/ClickHouse/ClickHouse/pull/64855) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#64976](https://github.com/ClickHouse/ClickHouse/issues/64976): Fix rare crash when table has TTL with subquery + database replicated + parallel replicas + analyzer. It's really rare, but please don't use TTLs with subqueries. [#64858](https://github.com/ClickHouse/ClickHouse/pull/64858) ([alesapin](https://github.com/alesapin)).
+* Backported in [#65074](https://github.com/ClickHouse/ClickHouse/issues/65074): Fix `ALTER MODIFY COMMENT` query that was broken for parameterized VIEWs in https://github.com/ClickHouse/ClickHouse/pull/54211. [#65031](https://github.com/ClickHouse/ClickHouse/pull/65031) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Backported in [#65179](https://github.com/ClickHouse/ClickHouse/issues/65179): Fix the `Unknown expression identifier` error for remote queries with `INTERPOLATE (alias)` (new analyzer). Fixes [#64636](https://github.com/ClickHouse/ClickHouse/issues/64636). [#65090](https://github.com/ClickHouse/ClickHouse/pull/65090) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65163](https://github.com/ClickHouse/ClickHouse/issues/65163): Fix pushing arithmetic operations out of aggregation. In the new analyzer, optimization was applied only once. Part of [#62245](https://github.com/ClickHouse/ClickHouse/issues/62245). [#65104](https://github.com/ClickHouse/ClickHouse/pull/65104) ([Dmitry Novik](https://github.com/novikd)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+
+* Backported in [#64882](https://github.com/ClickHouse/ClickHouse/issues/64882): This PR fixes an error when a user in a specific situation can escalate their privileges on the default database without necessary grants. [#64769](https://github.com/ClickHouse/ClickHouse/pull/64769) ([pufit](https://github.com/pufit)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#65002](https://github.com/ClickHouse/ClickHouse/issues/65002): Be more graceful with existing tables with `inverted` indexes. [#64656](https://github.com/ClickHouse/ClickHouse/pull/64656) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#65115](https://github.com/ClickHouse/ClickHouse/issues/65115): Adjust the `version_helper` and script to a new release scheme. [#64759](https://github.com/ClickHouse/ClickHouse/pull/64759) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#64796](https://github.com/ClickHouse/ClickHouse/issues/64796): Fix crash with DISTINCT and window functions. [#64767](https://github.com/ClickHouse/ClickHouse/pull/64767) ([Igor Nikonov](https://github.com/devcrafter)).
+
diff --git a/docs/changelogs/v24.5.3.5-stable.md b/docs/changelogs/v24.5.3.5-stable.md
new file mode 100644
index 00000000000..4606e58d0a4
--- /dev/null
+++ b/docs/changelogs/v24.5.3.5-stable.md
@@ -0,0 +1,14 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.5.3.5-stable (e0eb66f8e17) FIXME as compared to v24.5.2.34-stable (45589aeee49)
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#65227](https://github.com/ClickHouse/ClickHouse/issues/65227): Capture weak_ptr of ContextAccess for safety. [#65051](https://github.com/ClickHouse/ClickHouse/pull/65051) ([Alexander Gololobov](https://github.com/davenger)).
+* Backported in [#65219](https://github.com/ClickHouse/ClickHouse/issues/65219): Fix false positives leaky memory warnings in OpenSSL. [#65125](https://github.com/ClickHouse/ClickHouse/pull/65125) ([Robert Schulze](https://github.com/rschu1ze)).
+
diff --git a/docs/changelogs/v24.5.4.49-stable.md b/docs/changelogs/v24.5.4.49-stable.md
new file mode 100644
index 00000000000..56d497d5f59
--- /dev/null
+++ b/docs/changelogs/v24.5.4.49-stable.md
@@ -0,0 +1,41 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.5.4.49-stable (63b760955a0) FIXME as compared to v24.5.3.5-stable (e0eb66f8e17)
+
+#### Improvement
+* Backported in [#65886](https://github.com/ClickHouse/ClickHouse/issues/65886): Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#65304](https://github.com/ClickHouse/ClickHouse/issues/65304): Returned back the behaviour of how ClickHouse works and interprets Tuples in CSV format. This change effectively reverts https://github.com/ClickHouse/ClickHouse/pull/60994 and makes it available only under a few settings: `output_format_csv_serialize_tuple_into_separate_columns`, `input_format_csv_deserialize_separate_columns_into_tuple` and `input_format_csv_try_infer_strings_from_quoted_tuples`. [#65170](https://github.com/ClickHouse/ClickHouse/pull/65170) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Backported in [#65896](https://github.com/ClickHouse/ClickHouse/issues/65896): Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Backported in [#65287](https://github.com/ClickHouse/ClickHouse/issues/65287): Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#65374](https://github.com/ClickHouse/ClickHouse/issues/65374): Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Backported in [#65437](https://github.com/ClickHouse/ClickHouse/issues/65437): Forbid `QUALIFY` clause in the old analyzer. The old analyzer ignored `QUALIFY`, so it could lead to unexpected data removal in mutations. [#65356](https://github.com/ClickHouse/ClickHouse/pull/65356) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#65450](https://github.com/ClickHouse/ClickHouse/issues/65450): Use correct memory alignment for Distinct combinator. Previously, crash could happen because of invalid memory allocation when the combinator was used. [#65379](https://github.com/ClickHouse/ClickHouse/pull/65379) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#65712](https://github.com/ClickHouse/ClickHouse/issues/65712): Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#65681](https://github.com/ClickHouse/ClickHouse/issues/65681): Fix `duplicate alias` error for distributed queries with `ARRAY JOIN`. [#64226](https://github.com/ClickHouse/ClickHouse/pull/64226) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65331](https://github.com/ClickHouse/ClickHouse/issues/65331): Fix the crash loop when restoring from backup is blocked by creating an MV with a definer that hasn't been restored yet. [#64595](https://github.com/ClickHouse/ClickHouse/pull/64595) ([pufit](https://github.com/pufit)).
+* Backported in [#64835](https://github.com/ClickHouse/ClickHouse/issues/64835): Fix bug which could lead to non-working TTLs with expressions. [#64694](https://github.com/ClickHouse/ClickHouse/pull/64694) ([alesapin](https://github.com/alesapin)).
+* Backported in [#65542](https://github.com/ClickHouse/ClickHouse/issues/65542): Fix crash for `ALTER TABLE ... ON CLUSTER ... MODIFY SQL SECURITY`. [#64957](https://github.com/ClickHouse/ClickHouse/pull/64957) ([pufit](https://github.com/pufit)).
+* Backported in [#65580](https://github.com/ClickHouse/ClickHouse/issues/65580): Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Backported in [#65618](https://github.com/ClickHouse/ClickHouse/issues/65618): Fix possible infinite query duration in case of cyclic aliases. Fixes [#64849](https://github.com/ClickHouse/ClickHouse/issues/64849). [#65081](https://github.com/ClickHouse/ClickHouse/pull/65081) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65617](https://github.com/ClickHouse/ClickHouse/issues/65617): Fix aggregate function name rewriting in the new analyzer. [#65110](https://github.com/ClickHouse/ClickHouse/pull/65110) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#65732](https://github.com/ClickHouse/ClickHouse/issues/65732): Eliminate injective function in argument of functions `uniq*` recursively. This used to work correctly but was broken in the new analyzer. [#65140](https://github.com/ClickHouse/ClickHouse/pull/65140) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#65265](https://github.com/ClickHouse/ClickHouse/issues/65265): Fix the bug in Hashed and Hashed_Array dictionary short circuit evaluation, which may read uninitialized number, leading to various errors. [#65256](https://github.com/ClickHouse/ClickHouse/pull/65256) ([jsc0218](https://github.com/jsc0218)).
+* Backported in [#65663](https://github.com/ClickHouse/ClickHouse/issues/65663): Disable `non-intersecting-parts` optimization for queries with `FINAL` in case of `read-in-order` optimization was enabled. This could lead to an incorrect query result. As a workaround, disable `do_not_merge_across_partitions_select_final` and `split_parts_ranges_into_intersecting_and_non_intersecting_final` before this fix is merged. [#65505](https://github.com/ClickHouse/ClickHouse/pull/65505) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65788](https://github.com/ClickHouse/ClickHouse/issues/65788): Fixed bug in MergeJoin. Column in sparse serialisation might be treated as a column of its nested type though the required conversion wasn't performed. [#65632](https://github.com/ClickHouse/ClickHouse/pull/65632) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#65812](https://github.com/ClickHouse/ClickHouse/issues/65812): Fix invalid exceptions in function `parseDateTime` with `%F` and `%D` placeholders. [#65768](https://github.com/ClickHouse/ClickHouse/pull/65768) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#65828](https://github.com/ClickHouse/ClickHouse/issues/65828): Fix a bug in short circuit logic when old analyzer and dictGetOrDefault is used. [#65802](https://github.com/ClickHouse/ClickHouse/pull/65802) ([jsc0218](https://github.com/jsc0218)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#65412](https://github.com/ClickHouse/ClickHouse/issues/65412): Re-enable OpenSSL session caching. [#65111](https://github.com/ClickHouse/ClickHouse/pull/65111) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#65905](https://github.com/ClickHouse/ClickHouse/issues/65905): Fix bug with session closing in Keeper. [#65735](https://github.com/ClickHouse/ClickHouse/pull/65735) ([Antonio Andelic](https://github.com/antonio2368)).
+
diff --git a/docs/changelogs/v24.6.1.4423-stable.md b/docs/changelogs/v24.6.1.4423-stable.md
new file mode 100644
index 00000000000..f7af9cbaf8d
--- /dev/null
+++ b/docs/changelogs/v24.6.1.4423-stable.md
@@ -0,0 +1,735 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.6.1.4423-stable (dcced7c8478) FIXME as compared to v24.4.1.2088-stable (6d4b31322d1)
+
+#### Backward Incompatible Change
+* Enable asynchronous load of databases and tables by default. See the `async_load_databases` in config.xml. While this change is fully compatible, it can introduce a difference in behavior. When `async_load_databases` is false, as in the previous versions, the server will not accept connections until all tables are loaded. When `async_load_databases` is true, as in the new version, the server can accept connections before all the tables are loaded. If a query is made to a table that is not yet loaded, it will wait for the table's loading, which can take considerable time. It can change the behavior of the server if it is part of a large distributed system under a load balancer. In the first case, the load balancer can get a connection refusal and quickly failover to another server. In the second case, the load balancer can connect to a server that is still loading the tables, and the query will have a higher latency. Moreover, if many queries accumulate in the waiting state, it can lead to a "thundering herd" problem when they start processing simultaneously. This can make a difference only for highly loaded distributed backends. You can set the value of `async_load_databases` to false to avoid this problem. [#57695](https://github.com/ClickHouse/ClickHouse/pull/57695) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Some invalid queries will fail earlier during parsing. Note: disabled the support for inline KQL expressions (the experimental Kusto language) when they are put into a `kql` table function without a string literal, e.g. `kql(garbage | trash)` instead of `kql('garbage | trash')` or `kql($$garbage | trash$$)`. This feature was introduced unintentionally and should not exist. [#61500](https://github.com/ClickHouse/ClickHouse/pull/61500) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Renamed "inverted indexes" to "full-text indexes" which is a less technical / more user-friendly name. This also changes internal table metadata and breaks tables with existing (experimental) inverted indexes. Please make to drop such indexes before upgrade and re-create them after upgrade. [#62884](https://github.com/ClickHouse/ClickHouse/pull/62884) ([Robert Schulze](https://github.com/rschu1ze)).
+* Usage of functions `neighbor`, `runningAccumulate`, `runningDifferenceStartingWithFirstValue`, `runningDifference` deprecated (because it is error-prone). Proper window functions should be used instead. To enable them back, set `allow_deprecated_functions=1`. [#63132](https://github.com/ClickHouse/ClickHouse/pull/63132) ([Nikita Taranov](https://github.com/nickitat)).
+* Queries from `system.columns` will work faster if there is a large number of columns, but many databases or tables are not granted for `SHOW TABLES`. Note that in previous versions, if you grant `SHOW COLUMNS` to individual columns without granting `SHOW TABLES` to the corresponding tables, the `system.columns` table will show these columns, but in a new version, it will skip the table entirely. Remove trace log messages "Access granted" and "Access denied" that slowed down queries. [#63439](https://github.com/ClickHouse/ClickHouse/pull/63439) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Rework parallel processing in `Ordered` mode of storage `S3Queue`. This PR is backward incompatible for Ordered mode if you used settings `s3queue_processing_threads_num` or `s3queue_total_shards_num`. Setting `s3queue_total_shards_num` is deleted, previously it was allowed to use only under `s3queue_allow_experimental_sharded_mode`, which is now deprecated. A new setting is added - `s3queue_buckets`. [#64349](https://github.com/ClickHouse/ClickHouse/pull/64349) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* New functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` were added. Unlike the existing functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake`, the new functions are compatible with function `generateSnowflakeID`, i.e. they accept the snowflake IDs generated by `generateSnowflakeID` and produce snowflake IDs of the same type as `generateSnowflakeID` (i.e. `UInt64`). Furthermore, the new functions default to the UNIX epoch (aka. 1970-01-01), just like `generateSnowflakeID`. If necessary, a different epoch, e.g. Twitter's/X's epoch 2010-11-04 aka. 1288834974657 msec since UNIX epoch, can be passed. The old conversion functions are deprecated and will be removed after a transition period: to use them regardless, enable setting `allow_deprecated_snowflake_conversion_functions`. [#64948](https://github.com/ClickHouse/ClickHouse/pull/64948) ([Robert Schulze](https://github.com/rschu1ze)).
+
+#### New Feature
+* Provide support for AzureBlobStorage function in ClickHouse server to use Azure Workload identity to authenticate against Azure blob storage. If `use_workload_identity` parameter is set in config, [workload identity](https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity#authenticate-azure-hosted-applications) is used for authentication. [#57881](https://github.com/ClickHouse/ClickHouse/pull/57881) ([Vinay Suryadevara](https://github.com/vinay92-ch)).
+* Introduce bulk loading to StorageEmbeddedRocksDB by creating and ingesting SST file instead of relying on rocksdb build-in memtable. This help to increase importing speed, especially for long-running insert query to StorageEmbeddedRocksDB tables. Also, introduce `StorageEmbeddedRocksDB` table settings. [#59163](https://github.com/ClickHouse/ClickHouse/pull/59163) ([Duc Canh Le](https://github.com/canhld94)).
+* Introduce statistics of type "number of distinct values". [#59357](https://github.com/ClickHouse/ClickHouse/pull/59357) ([Han Fei](https://github.com/hanfei1991)).
+* User can now parse CRLF with TSV format using a setting `input_format_tsv_crlf_end_of_line`. Closes [#56257](https://github.com/ClickHouse/ClickHouse/issues/56257). [#59747](https://github.com/ClickHouse/ClickHouse/pull/59747) ([Shaun Struwig](https://github.com/Blargian)).
+* Add Hilbert Curve encode and decode functions. [#60156](https://github.com/ClickHouse/ClickHouse/pull/60156) ([Artem Mustafin](https://github.com/Artemmm91)).
+* Adds the Form Format to read/write a single record in the application/x-www-form-urlencoded format. [#60199](https://github.com/ClickHouse/ClickHouse/pull/60199) ([Shaun Struwig](https://github.com/Blargian)).
+* Added possibility to compress in CROSS JOIN. [#60459](https://github.com/ClickHouse/ClickHouse/pull/60459) ([p1rattttt](https://github.com/p1rattttt)).
+* New setting `input_format_force_null_for_omitted_fields` that forces NULL values for omitted fields. [#60887](https://github.com/ClickHouse/ClickHouse/pull/60887) ([Constantine Peresypkin](https://github.com/pkit)).
+* Support join with inequal conditions which involve columns from both left and right table. e.g. `t1.y < t2.y`. To enable, `SET allow_experimental_join_condition = 1`. [#60920](https://github.com/ClickHouse/ClickHouse/pull/60920) ([lgbo](https://github.com/lgbo-ustc)).
+* Earlier our s3 storage and s3 table function didn't support selecting from archive files. I created a solution that allows to iterate over files inside archives in S3. [#62259](https://github.com/ClickHouse/ClickHouse/pull/62259) ([Daniil Ivanik](https://github.com/divanik)).
+* Support for conditional function `clamp`. [#62377](https://github.com/ClickHouse/ClickHouse/pull/62377) ([skyoct](https://github.com/skyoct)).
+* Add npy output format. [#62430](https://github.com/ClickHouse/ClickHouse/pull/62430) ([豪肥肥](https://github.com/HowePa)).
+* Added support for reading LINESTRING geometry in WKT format using function `readWKTLineString`. [#62519](https://github.com/ClickHouse/ClickHouse/pull/62519) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Added SQL functions `generateUUIDv7`, `generateUUIDv7ThreadMonotonic`, `generateUUIDv7NonMonotonic` (with different monotonicity/performance trade-offs) to generate version 7 UUIDs aka. timestamp-based UUIDs with random component. Also added a new function `UUIDToNum` to extract bytes from a UUID and a new function `UUIDv7ToDateTime` to extract timestamp component from a UUID version 7. [#62852](https://github.com/ClickHouse/ClickHouse/pull/62852) ([Alexey Petrunyaka](https://github.com/pet74alex)).
+* Implement Dynamic data type that allows to store values of any type inside it without knowing all of them in advance. Dynamic type is available under a setting `allow_experimental_dynamic_type`. [#63058](https://github.com/ClickHouse/ClickHouse/pull/63058) ([Kruglov Pavel](https://github.com/Avogar)).
+* Allow to attach parts from a different disk. [#63087](https://github.com/ClickHouse/ClickHouse/pull/63087) ([Unalian](https://github.com/Unalian)).
+* Allow proxy to be bypassed for hosts specified in `no_proxy` env variable and ClickHouse proxy configuration. [#63314](https://github.com/ClickHouse/ClickHouse/pull/63314) ([Arthur Passos](https://github.com/arthurpassos)).
+* Introduce bulk loading to StorageEmbeddedRocksDB by creating and ingesting SST file instead of relying on rocksdb build-in memtable. This help to increase importing speed, especially for long-running insert query to StorageEmbeddedRocksDB tables. Also, introduce StorageEmbeddedRocksDB table settings. [#63324](https://github.com/ClickHouse/ClickHouse/pull/63324) ([Duc Canh Le](https://github.com/canhld94)).
+* Raw as a synonym for TSVRaw. [#63394](https://github.com/ClickHouse/ClickHouse/pull/63394) ([Unalian](https://github.com/Unalian)).
+* Added possibility to do cross join in temporary file if size exceeds limits. [#63432](https://github.com/ClickHouse/ClickHouse/pull/63432) ([p1rattttt](https://github.com/p1rattttt)).
+* Added a new table function `loop` to support returning query results in an infinite loop. [#63452](https://github.com/ClickHouse/ClickHouse/pull/63452) ([Sariel](https://github.com/sarielwxm)).
+* Added new SQL functions `generateSnowflakeID` for generating Twitter-style Snowflake IDs. [#63577](https://github.com/ClickHouse/ClickHouse/pull/63577) ([Danila Puzov](https://github.com/kazalika)).
+* Add the ability to reshuffle rows during insert to optimize for size without violating the order set by `PRIMARY KEY`. It's controlled by the setting `optimize_row_order` (off by default). [#63578](https://github.com/ClickHouse/ClickHouse/pull/63578) ([Igor Markelov](https://github.com/ElderlyPassionFruit)).
+* On Linux and MacOS, if the program has STDOUT redirected to a file with a compression extension, use the corresponding compression method instead of nothing (making it behave similarly to `INTO OUTFILE` ). [#63662](https://github.com/ClickHouse/ClickHouse/pull/63662) ([v01dXYZ](https://github.com/v01dXYZ)).
+* Added `merge_workload` and `mutation_workload` settings to regulate how resources are utilized and shared between merges, mutations and other workloads. [#64061](https://github.com/ClickHouse/ClickHouse/pull/64061) ([Sergei Trifonov](https://github.com/serxa)).
+* Change warning on high number of attached tables to differentiate tables, views and dictionaries. [#64180](https://github.com/ClickHouse/ClickHouse/pull/64180) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)).
+* Add support for comparing IPv4 and IPv6 types using the `=` operator. [#64292](https://github.com/ClickHouse/ClickHouse/pull/64292) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)).
+* Allow to store named collections in zookeeper. [#64574](https://github.com/ClickHouse/ClickHouse/pull/64574) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Support decimal arguments in binary math functions (pow(), atan2(), max2, min2(), hypot(). [#64582](https://github.com/ClickHouse/ClickHouse/pull/64582) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
+* Add support for index analysis over `hilbertEncode`. [#64662](https://github.com/ClickHouse/ClickHouse/pull/64662) ([Artem Mustafin](https://github.com/Artemmm91)).
+* Added SQL functions `parseReadableSize` (along with `OrNull` and `OrZero` variants). [#64742](https://github.com/ClickHouse/ClickHouse/pull/64742) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)).
+* Add server settings `max_table_num_to_throw` and `max_database_num_to_throw` to limit the number of databases or tables on `CREATE` queries. [#64781](https://github.com/ClickHouse/ClickHouse/pull/64781) ([Xu Jia](https://github.com/XuJia0210)).
+* Add _time virtual column to file alike storages (s3/file/hdfs/url/azureBlobStorage). [#64947](https://github.com/ClickHouse/ClickHouse/pull/64947) ([Ilya Golshtein](https://github.com/ilejn)).
+* Introduced new functions `base64URLEncode`, `base64URLDecode` and `tryBase64URLDecode`. [#64991](https://github.com/ClickHouse/ClickHouse/pull/64991) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
+* Add new function `editDistanceUTF8`, which calculates the [edit distance](https://en.wikipedia.org/wiki/Edit_distance) between two UTF8 strings. [#65269](https://github.com/ClickHouse/ClickHouse/pull/65269) ([LiuNeng](https://github.com/liuneng1994)).
+
+#### Performance Improvement
+* Skip merging of newly created projection blocks during `INSERT`-s. [#59405](https://github.com/ClickHouse/ClickHouse/pull/59405) ([Nikita Taranov](https://github.com/nickitat)).
+* Add a native parquet reader, which can read parquet binary to ClickHouse Columns directly. It's controlled by the setting `input_format_parquet_use_native_reader` (disabled by default). [#60361](https://github.com/ClickHouse/ClickHouse/pull/60361) ([ZhiHong Zhang](https://github.com/copperybean)).
+* Reduce the number of virtual function calls in ColumnNullable::size(). [#60556](https://github.com/ClickHouse/ClickHouse/pull/60556) ([HappenLee](https://github.com/HappenLee)).
+* Process string functions XXXUTF8 'asciily' if input strings are all ascii chars. Inspired by https://github.com/apache/doris/pull/29799. Overall speed up by 1.07x~1.62x. Notice that peak memory usage had been decreased in some cases. [#61632](https://github.com/ClickHouse/ClickHouse/pull/61632) ([李扬](https://github.com/taiyang-li)).
+* Improved performance of selection (`{}`) globs in StorageS3. [#62120](https://github.com/ClickHouse/ClickHouse/pull/62120) ([Andrey Zvonov](https://github.com/zvonand)).
+* HostResolver has each IP address several times. If remote host has several IPs and by some reason (firewall rules for example) access on some IPs allowed and on others forbidden, than only first record of forbidden IPs marked as failed, and in each try these IPs have a chance to be chosen (and failed again). Even if fix this, every 120 seconds DNS cache dropped, and IPs can be chosen again. [#62652](https://github.com/ClickHouse/ClickHouse/pull/62652) ([Anton Ivashkin](https://github.com/ianton-ru)).
+* Speedup `splitByRegexp` when the regular expression argument is a single-character. [#62696](https://github.com/ClickHouse/ClickHouse/pull/62696) ([Robert Schulze](https://github.com/rschu1ze)).
+* Speed up FixedHashTable by keeping track of the min and max keys used. This allows to reduce the number of cells that need to be verified. [#62746](https://github.com/ClickHouse/ClickHouse/pull/62746) ([Jiebin Sun](https://github.com/jiebinn)).
+* Add a new configuration`prefer_merge_sort_block_bytes` to control the memory usage and speed up sorting 2 times when merging when there are many columns. [#62904](https://github.com/ClickHouse/ClickHouse/pull/62904) ([LiuNeng](https://github.com/liuneng1994)).
+* `clickhouse-local` will start faster. In previous versions, it was not deleting temporary directories by mistake. Now it will. This closes [#62941](https://github.com/ClickHouse/ClickHouse/issues/62941). [#63074](https://github.com/ClickHouse/ClickHouse/pull/63074) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Micro-optimizations for the new analyzer. [#63429](https://github.com/ClickHouse/ClickHouse/pull/63429) ([Raúl Marín](https://github.com/Algunenano)).
+* Index analysis will work if `DateTime` is compared to `DateTime64`. This closes [#63441](https://github.com/ClickHouse/ClickHouse/issues/63441). [#63443](https://github.com/ClickHouse/ClickHouse/pull/63443) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Index analysis will work if `DateTime` is compared to `DateTime64`. This closes [#63441](https://github.com/ClickHouse/ClickHouse/issues/63441). [#63532](https://github.com/ClickHouse/ClickHouse/pull/63532) ([Raúl Marín](https://github.com/Algunenano)).
+* Optimize the resolution of in(LowCardinality, ConstantSet). [#64060](https://github.com/ClickHouse/ClickHouse/pull/64060) ([Zhiguo Zhou](https://github.com/ZhiguoZh)).
+* Speed up indices of type `set` a little (around 1.5 times) by removing garbage. [#64098](https://github.com/ClickHouse/ClickHouse/pull/64098) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Use a thread pool to initialize and destroy hash tables inside `ConcurrentHashJoin`. [#64241](https://github.com/ClickHouse/ClickHouse/pull/64241) ([Nikita Taranov](https://github.com/nickitat)).
+* Optimized vertical merges in tables with sparse columns. [#64311](https://github.com/ClickHouse/ClickHouse/pull/64311) ([Anton Popov](https://github.com/CurtizJ)).
+* Enabled prefetches of data from remote filesystem during vertical merges. It improves latency of vertical merges in tables with data stored on remote filesystem. [#64314](https://github.com/ClickHouse/ClickHouse/pull/64314) ([Anton Popov](https://github.com/CurtizJ)).
+* Reduce redundant calls to `isDefault()` of `ColumnSparse::filter` to improve performance. [#64426](https://github.com/ClickHouse/ClickHouse/pull/64426) ([Jiebin Sun](https://github.com/jiebinn)).
+* Speedup `find_super_nodes` and `find_big_family` keeper-client commands by making multiple asynchronous getChildren requests. [#64628](https://github.com/ClickHouse/ClickHouse/pull/64628) ([Alexander Gololobov](https://github.com/davenger)).
+* Improve function least/greatest for nullable numberic type arguments. [#64668](https://github.com/ClickHouse/ClickHouse/pull/64668) ([KevinyhZou](https://github.com/KevinyhZou)).
+* Allow merging two consequent `FilterSteps` of a query plan. This improves filter-push-down optimization if the filter condition can be pushed down from the parent step. [#64760](https://github.com/ClickHouse/ClickHouse/pull/64760) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Remove bad optimization in vertical final implementation and re-enable vertical final algorithm by default. [#64783](https://github.com/ClickHouse/ClickHouse/pull/64783) ([Duc Canh Le](https://github.com/canhld94)).
+* Remove ALIAS nodes from the filter expression. This slightly improves performance for queries with `PREWHERE` (with new analyzer). [#64793](https://github.com/ClickHouse/ClickHouse/pull/64793) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix performance regression in cross join introduced in [#60459](https://github.com/ClickHouse/ClickHouse/issues/60459) (24.5). [#65243](https://github.com/ClickHouse/ClickHouse/pull/65243) ([Nikita Taranov](https://github.com/nickitat)).
+
+#### Improvement
+* Support empty tuples. [#55061](https://github.com/ClickHouse/ClickHouse/pull/55061) ([Amos Bird](https://github.com/amosbird)).
+* Hot reload storage policy for distributed tables when adding a new disk. [#58285](https://github.com/ClickHouse/ClickHouse/pull/58285) ([Duc Canh Le](https://github.com/canhld94)).
+* Maps can now have `Float32`, `Float64`, `Array(T)`, `Map(K,V)` and `Tuple(T1, T2, ...)` as keys. Closes [#54537](https://github.com/ClickHouse/ClickHouse/issues/54537). [#59318](https://github.com/ClickHouse/ClickHouse/pull/59318) ([李扬](https://github.com/taiyang-li)).
+* Avoid possible deadlock during MergeTree index analysis when scheduling threads in a saturated service. [#59427](https://github.com/ClickHouse/ClickHouse/pull/59427) ([Sean Haynes](https://github.com/seandhaynes)).
+* Multiline strings with border preservation and column width change. [#59940](https://github.com/ClickHouse/ClickHouse/pull/59940) ([Volodyachan](https://github.com/Volodyachan)).
+* Make rabbitmq nack broken messages. Closes [#45350](https://github.com/ClickHouse/ClickHouse/issues/45350). [#60312](https://github.com/ClickHouse/ClickHouse/pull/60312) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Support partial trivial count optimization when the query filter is able to select exact ranges from merge tree tables. [#60463](https://github.com/ClickHouse/ClickHouse/pull/60463) ([Amos Bird](https://github.com/amosbird)).
+* Fix a crash in asynchronous stack unwinding (such as when using the sampling query profiler) while interpreting debug info. This closes [#60460](https://github.com/ClickHouse/ClickHouse/issues/60460). [#60468](https://github.com/ClickHouse/ClickHouse/pull/60468) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Reduce max memory usage of multithreaded `INSERT`s by collecting chunks of multiple threads in a single transform. [#61047](https://github.com/ClickHouse/ClickHouse/pull/61047) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Distinct messages for s3 error 'no key' for cases disk and storage. [#61108](https://github.com/ClickHouse/ClickHouse/pull/61108) ([Sema Checherinda](https://github.com/CheSema)).
+* Less contention in filesystem cache (part 4). Allow to keep filesystem cache not filled to the limit by doing additional eviction in the background (controlled by `keep_free_space_size(elements)_ratio`). This allows to release pressure from space reservation for queries (on `tryReserve` method). Also this is done in a lock free way as much as possible, e.g. should not block normal cache usage. [#61250](https://github.com/ClickHouse/ClickHouse/pull/61250) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* The progress bar will work for trivial queries with LIMIT from `system.zeros`, `system.zeros_mt` (it already works for `system.numbers` and `system.numbers_mt`), and the `generateRandom` table function. As a bonus, if the total number of records is greater than the `max_rows_to_read` limit, it will throw an exception earlier. This closes [#58183](https://github.com/ClickHouse/ClickHouse/issues/58183). [#61823](https://github.com/ClickHouse/ClickHouse/pull/61823) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* YAML Merge Key support. [#62685](https://github.com/ClickHouse/ClickHouse/pull/62685) ([Azat Khuzhin](https://github.com/azat)).
+* Enhance error message when non-deterministic function is used with Replicated source. [#62896](https://github.com/ClickHouse/ClickHouse/pull/62896) ([Grégoire Pineau](https://github.com/lyrixx)).
+* Fix interserver secret for Distributed over Distributed from `remote`. [#63013](https://github.com/ClickHouse/ClickHouse/pull/63013) ([Azat Khuzhin](https://github.com/azat)).
+* Allow using `clickhouse-local` and its shortcuts `clickhouse` and `ch` with a query or queries file as a positional argument. Examples: `ch "SELECT 1"`, `ch --param_test Hello "SELECT {test:String}"`, `ch query.sql`. This closes [#62361](https://github.com/ClickHouse/ClickHouse/issues/62361). [#63081](https://github.com/ClickHouse/ClickHouse/pull/63081) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Support configuration substitutions from YAML files. [#63106](https://github.com/ClickHouse/ClickHouse/pull/63106) ([Eduard Karacharov](https://github.com/korowa)).
+* Reduce the memory usage when using Azure object storage by using fixed memory allocation, avoiding the allocation of an extra buffer. [#63160](https://github.com/ClickHouse/ClickHouse/pull/63160) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Add TTL information in system parts_columns table. [#63200](https://github.com/ClickHouse/ClickHouse/pull/63200) ([litlig](https://github.com/litlig)).
+* Keep previous data in terminal after picking from skim suggestions. [#63261](https://github.com/ClickHouse/ClickHouse/pull/63261) ([FlameFactory](https://github.com/FlameFactory)).
+* Width of fields now correctly calculate, ignoring ANSI escape sequences. [#63270](https://github.com/ClickHouse/ClickHouse/pull/63270) ([Shaun Struwig](https://github.com/Blargian)).
+* Enable plain_rewritable metadata for local and Azure (azure_blob_storage) object storages. [#63365](https://github.com/ClickHouse/ClickHouse/pull/63365) ([Julia Kartseva](https://github.com/jkartseva)).
+* Support English-style Unicode quotes, e.g. “Hello”, ‘world’. This is questionable in general but helpful when you type your query in a word processor, such as Google Docs. This closes [#58634](https://github.com/ClickHouse/ClickHouse/issues/58634). [#63381](https://github.com/ClickHouse/ClickHouse/pull/63381) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Allowed to create MaterializedMySQL database without connection to MySQL. [#63397](https://github.com/ClickHouse/ClickHouse/pull/63397) ([Kirill](https://github.com/kirillgarbar)).
+* Remove copying data when writing to filesystem cache. [#63401](https://github.com/ClickHouse/ClickHouse/pull/63401) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Update the usage of error code `NUMBER_OF_ARGUMENTS_DOESNT_MATCH` by more accurate error codes when appropriate. [#63406](https://github.com/ClickHouse/ClickHouse/pull/63406) ([Yohann Jardin](https://github.com/yohannj)).
+* Several minor corner case fixes to proxy support & tunneling. [#63427](https://github.com/ClickHouse/ClickHouse/pull/63427) ([Arthur Passos](https://github.com/arthurpassos)).
+* `os_user` and `client_hostname` are now correctly set up for queries for command line suggestions in clickhouse-client. This closes [#63430](https://github.com/ClickHouse/ClickHouse/issues/63430). [#63433](https://github.com/ClickHouse/ClickHouse/pull/63433) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fixed tabulation from line numbering, correct handling of length when moving a line if the value has a tab, added tests. [#63493](https://github.com/ClickHouse/ClickHouse/pull/63493) ([Volodyachan](https://github.com/Volodyachan)).
+* Add this `aggregate_function_group_array_has_limit_size`setting to support discarding data in some scenarios. [#63516](https://github.com/ClickHouse/ClickHouse/pull/63516) ([zhongyuankai](https://github.com/zhongyuankai)).
+* Automatically mark a replica of Replicated database as lost and start recovery if some DDL task fails more than `max_retries_before_automatic_recovery` (100 by default) times in a row with the same error. Also, fixed a bug that could cause skipping DDL entries when an exception is thrown during an early stage of entry execution. [#63549](https://github.com/ClickHouse/ClickHouse/pull/63549) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Add `http_response_headers` setting to support custom response headers in custom HTTP handlers. [#63562](https://github.com/ClickHouse/ClickHouse/pull/63562) ([Grigorii](https://github.com/GSokol)).
+* Automatically correct `max_block_size=0` to default value. [#63587](https://github.com/ClickHouse/ClickHouse/pull/63587) ([Antonio Andelic](https://github.com/antonio2368)).
+* Account failed files in `s3queue_tracked_file_ttl_sec` and `s3queue_traked_files_limit` for `StorageS3Queue`. [#63638](https://github.com/ClickHouse/ClickHouse/pull/63638) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add a build_id ALIAS column to trace_log to facilitate auto renaming upon detecting binary changes. This is to address [#52086](https://github.com/ClickHouse/ClickHouse/issues/52086). [#63656](https://github.com/ClickHouse/ClickHouse/pull/63656) ([Zimu Li](https://github.com/woodlzm)).
+* Enable truncate operation for object storage disks. [#63693](https://github.com/ClickHouse/ClickHouse/pull/63693) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Improve io_uring resubmits visibility. Rename profile event `IOUringSQEsResubmits` -> `IOUringSQEsResubmitsAsync` and add a new one `IOUringSQEsResubmitsSync`. [#63699](https://github.com/ClickHouse/ClickHouse/pull/63699) ([Tomer Shafir](https://github.com/tomershafir)).
+* Introduce assertions to verify all functions are called with columns of the right size. [#63723](https://github.com/ClickHouse/ClickHouse/pull/63723) ([Raúl Marín](https://github.com/Algunenano)).
+* The loading of the keywords list is now dependent on the server revision and will be disabled for the old versions of ClickHouse server. CC @azat. [#63786](https://github.com/ClickHouse/ClickHouse/pull/63786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* `SHOW CREATE TABLE` executed on top of system tables will now show the super handy comment unique for each table which will explain why this table is needed. [#63788](https://github.com/ClickHouse/ClickHouse/pull/63788) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Allow trailing commas in the columns list in the INSERT query. For example, `INSERT INTO test (a, b, c, ) VALUES ...`. [#63803](https://github.com/ClickHouse/ClickHouse/pull/63803) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Better exception messages for the `Regexp` format. [#63804](https://github.com/ClickHouse/ClickHouse/pull/63804) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Allow trailing commas in the `Values` format. For example, this query is allowed: `INSERT INTO test (a, b, c) VALUES (4, 5, 6,);`. [#63810](https://github.com/ClickHouse/ClickHouse/pull/63810) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Clickhouse disks have to read server setting to obtain actual metadata format version. [#63831](https://github.com/ClickHouse/ClickHouse/pull/63831) ([Sema Checherinda](https://github.com/CheSema)).
+* Disable pretty format restrictions (`output_format_pretty_max_rows`/`output_format_pretty_max_value_width`) when stdout is not TTY. [#63942](https://github.com/ClickHouse/ClickHouse/pull/63942) ([Azat Khuzhin](https://github.com/azat)).
+* Exception handling now works when ClickHouse is used inside AWS Lambda. Author: [Alexey Coolnev](https://github.com/acoolnev). [#64014](https://github.com/ClickHouse/ClickHouse/pull/64014) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Throw `CANNOT_DECOMPRESS` instread of `CORRUPTED_DATA` on invalid compressed data passed via HTTP. [#64036](https://github.com/ClickHouse/ClickHouse/pull/64036) ([vdimir](https://github.com/vdimir)).
+* A tip for a single large number in Pretty formats now works for Nullable and LowCardinality. This closes [#61993](https://github.com/ClickHouse/ClickHouse/issues/61993). [#64084](https://github.com/ClickHouse/ClickHouse/pull/64084) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Now backups with azure blob storage will use multicopy. [#64116](https://github.com/ClickHouse/ClickHouse/pull/64116) ([alesapin](https://github.com/alesapin)).
+* Added a new setting, `metadata_keep_free_space_bytes` to keep free space on the metadata storage disk. [#64128](https://github.com/ClickHouse/ClickHouse/pull/64128) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Add metrics, logs, and thread names around parts filtering with indices. [#64130](https://github.com/ClickHouse/ClickHouse/pull/64130) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Allow to use native copy for azure even with different containers. [#64154](https://github.com/ClickHouse/ClickHouse/pull/64154) ([alesapin](https://github.com/alesapin)).
+* Add metrics to track the number of directories created and removed by the plain_rewritable metadata storage, and the number of entries in the local-to-remote in-memory map. [#64175](https://github.com/ClickHouse/ClickHouse/pull/64175) ([Julia Kartseva](https://github.com/jkartseva)).
+* Finally enable native copy for azure. [#64182](https://github.com/ClickHouse/ClickHouse/pull/64182) ([alesapin](https://github.com/alesapin)).
+* Ignore `allow_suspicious_primary_key` on `ATTACH` and verify on `ALTER`. [#64202](https://github.com/ClickHouse/ClickHouse/pull/64202) ([Azat Khuzhin](https://github.com/azat)).
+* The query cache now considers identical queries with different settings as different. This increases robustness in cases where different settings (e.g. `limit` or `additional_table_filters`) would affect the query result. [#64205](https://github.com/ClickHouse/ClickHouse/pull/64205) ([Robert Schulze](https://github.com/rschu1ze)).
+* Better Exception Message in Delete Table with Projection, users can understand the error and the steps should be taken. [#64212](https://github.com/ClickHouse/ClickHouse/pull/64212) ([jsc0218](https://github.com/jsc0218)).
+* Support the non standard error code `QpsLimitExceeded` in object storage as a retryable error. [#64225](https://github.com/ClickHouse/ClickHouse/pull/64225) ([Sema Checherinda](https://github.com/CheSema)).
+* Forbid converting a MergeTree table to replicated if the zookeeper path for this table already exists. [#64244](https://github.com/ClickHouse/ClickHouse/pull/64244) ([Kirill](https://github.com/kirillgarbar)).
+* If "replica group" is configured for a `Replicated` database, automatically create a cluster that includes replicas from all groups. [#64312](https://github.com/ClickHouse/ClickHouse/pull/64312) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Added settings to disable materialization of skip indexes and statistics on inserts (`materialize_skip_indexes_on_insert` and `materialize_statistics_on_insert`). [#64391](https://github.com/ClickHouse/ClickHouse/pull/64391) ([Anton Popov](https://github.com/CurtizJ)).
+* Use the allocated memory size to calculate the row group size and reduce the peak memory of the parquet writer in single-threaded mode. [#64424](https://github.com/ClickHouse/ClickHouse/pull/64424) ([LiuNeng](https://github.com/liuneng1994)).
+* Added new configuration input_format_parquet_prefer_block_bytes to control the average output block bytes, and modified the default value of input_format_parquet_max_block_size to 65409. [#64427](https://github.com/ClickHouse/ClickHouse/pull/64427) ([LiuNeng](https://github.com/liuneng1994)).
+* Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)).
+* Settings from user config doesn't affect merges and mutations for MergeTree on top of object storage. [#64456](https://github.com/ClickHouse/ClickHouse/pull/64456) ([alesapin](https://github.com/alesapin)).
+* Setting `replace_long_file_name_to_hash` is enabled by default for `MergeTree` tables. [#64457](https://github.com/ClickHouse/ClickHouse/pull/64457) ([Anton Popov](https://github.com/CurtizJ)).
+* Improve the iterator of sparse column to reduce call of size(). [#64497](https://github.com/ClickHouse/ClickHouse/pull/64497) ([Jiebin Sun](https://github.com/jiebinn)).
+* Update condition to use copy for azure blob storage. [#64518](https://github.com/ClickHouse/ClickHouse/pull/64518) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Support the non standard error code `TotalQpsLimitExceeded` in object storage as a retryable error. [#64520](https://github.com/ClickHouse/ClickHouse/pull/64520) ([Sema Checherinda](https://github.com/CheSema)).
+* Optimized memory usage of vertical merges for tables with high number of skip indexes. [#64580](https://github.com/ClickHouse/ClickHouse/pull/64580) ([Anton Popov](https://github.com/CurtizJ)).
+* Introduced two additional columns in the `system.query_log`: `used_privileges` and `missing_privileges`. `used_privileges` is populated with the privileges that were checked during query execution, and `missing_privileges` contains required privileges that are missing. [#64597](https://github.com/ClickHouse/ClickHouse/pull/64597) ([Alexey Katsman](https://github.com/alexkats)).
+* Add settings `parallel_replicas_custom_key_range_lower` and `parallel_replicas_custom_key_range_upper` to control how parallel replicas with dynamic shards parallelizes queries when using a range filter. [#64604](https://github.com/ClickHouse/ClickHouse/pull/64604) ([josh-hildred](https://github.com/josh-hildred)).
+* Updated Advanced Dashboard for both open-source and ClickHouse Cloud versions to include a chart for 'Maximum concurrent network connections'. [#64610](https://github.com/ClickHouse/ClickHouse/pull/64610) ([Thom O'Connor](https://github.com/thomoco)).
+* The second argument (scale) of functions `round()`, `roundBankers()`, `floor()`, `ceil()` and `trunc()` can now be non-const. [#64798](https://github.com/ClickHouse/ClickHouse/pull/64798) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
+* Improve progress report on zeros_mt and generateRandom. [#64804](https://github.com/ClickHouse/ClickHouse/pull/64804) ([Raúl Marín](https://github.com/Algunenano)).
+* Add an asynchronous metric jemalloc.profile.active to show whether sampling is currently active. This is an activation mechanism in addition to prof.active; both must be active for the calling thread to sample. [#64842](https://github.com/ClickHouse/ClickHouse/pull/64842) ([Unalian](https://github.com/Unalian)).
+* Support statistics with ReplicatedMergeTree. [#64934](https://github.com/ClickHouse/ClickHouse/pull/64934) ([Han Fei](https://github.com/hanfei1991)).
+* Don't mark of `allow_experimental_join_condition` as IMPORTANT. This may have prevented distributed queries in a mixed versions cluster from being executed successfully. [#65008](https://github.com/ClickHouse/ClickHouse/pull/65008) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Backported in [#65716](https://github.com/ClickHouse/ClickHouse/issues/65716): `StorageS3Queue` related fixes and improvements. Deduce a default value of `s3queue_processing_threads_num` according to the number of physical cpu cores on the server (instead of the previous default value as 1). Set default value of `s3queue_loading_retries` to 10. Fix possible vague "Uncaught exception" in exception column of `system.s3queue`. Do not increment retry count on `MEMORY_LIMIT_EXCEEDED` exception. Move files commit to a stage after insertion into table fully finished to avoid files being commited while not inserted. Add settings `s3queue_max_processed_files_before_commit`, `s3queue_max_processed_rows_before_commit`, `s3queue_max_processed_bytes_before_commit`, `s3queue_max_processing_time_sec_before_commit`, to better control commit and flush time. [#65046](https://github.com/ClickHouse/ClickHouse/pull/65046) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Added server Asynchronous metrics `DiskGetObjectThrottler*` and `DiskGetObjectThrottler*` reflecting request per second rate limit defined with `s3_max_get_rps` and `s3_max_put_rps` disk settings and currently available number of requests that could be sent without hitting throttling limit on the disk. Metrics are defined for every disk that has a configured limit. [#65050](https://github.com/ClickHouse/ClickHouse/pull/65050) ([Sergei Trifonov](https://github.com/serxa)).
+* Added a setting `output_format_pretty_display_footer_column_names` which when enabled displays column names at the end of the table for long tables (50 rows by default), with the threshold value for minimum number of rows controlled by `output_format_pretty_display_footer_column_names_min_rows`. [#65144](https://github.com/ClickHouse/ClickHouse/pull/65144) ([Shaun Struwig](https://github.com/Blargian)).
+* Returned back the behaviour of how ClickHouse works and interprets Tuples in CSV format. This change effectively reverts https://github.com/ClickHouse/ClickHouse/pull/60994 and makes it available only under a few settings: `output_format_csv_serialize_tuple_into_separate_columns`, `input_format_csv_deserialize_separate_columns_into_tuple` and `input_format_csv_try_infer_strings_from_quoted_tuples`. [#65170](https://github.com/ClickHouse/ClickHouse/pull/65170) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Initialize global trace collector for Poco::ThreadPool (needed for keeper, etc). [#65239](https://github.com/ClickHouse/ClickHouse/pull/65239) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add validation when creating a user with bcrypt_hash. [#65242](https://github.com/ClickHouse/ClickHouse/pull/65242) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Fix a permission error where a user in a specific situation can escalate their privileges on the default database without necessary grants. [#64769](https://github.com/ClickHouse/ClickHouse/pull/64769) ([pufit](https://github.com/pufit)).
+* Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Forbid `QUALIFY` clause in the old analyzer. The old analyzer ignored `QUALIFY`, so it could lead to unexpected data removal in mutations. [#65356](https://github.com/ClickHouse/ClickHouse/pull/65356) ([Dmitry Novik](https://github.com/novikd)).
+* Use correct memory alignment for Distinct combinator. Previously, crash could happen because of invalid memory allocation when the combinator was used. [#65379](https://github.com/ClickHouse/ClickHouse/pull/65379) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#65846](https://github.com/ClickHouse/ClickHouse/issues/65846): Check cyclic dependencies on CREATE/REPLACE/RENAME/EXCHANGE queries and throw an exception if there is a cyclic dependency. Previously such cyclic dependencies could lead to a deadlock during server startup. Closes [#65355](https://github.com/ClickHouse/ClickHouse/issues/65355). Also fix some bugs in dependencies creation. [#65405](https://github.com/ClickHouse/ClickHouse/pull/65405) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#65714](https://github.com/ClickHouse/ClickHouse/issues/65714): Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Fix making backup when multiple shards are used. This PR fixes [#56566](https://github.com/ClickHouse/ClickHouse/issues/56566). [#57684](https://github.com/ClickHouse/ClickHouse/pull/57684) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix passing projections/indexes from CREATE query into inner table of MV. [#59183](https://github.com/ClickHouse/ClickHouse/pull/59183) ([Azat Khuzhin](https://github.com/azat)).
+* Fix boundRatio incorrect merge. [#60532](https://github.com/ClickHouse/ClickHouse/pull/60532) ([Tao Wang](https://github.com/wangtZJU)).
+* Fix crash when using some functions with low-cardinality columns. [#61966](https://github.com/ClickHouse/ClickHouse/pull/61966) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fixed 'set' skip index not working with IN and indexHint(). [#62083](https://github.com/ClickHouse/ClickHouse/pull/62083) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix queries with FINAL give wrong result when table does not use adaptive granularity. [#62432](https://github.com/ClickHouse/ClickHouse/pull/62432) ([Duc Canh Le](https://github.com/canhld94)).
+* Improve the detection of cgroups v2 memory controller in unusual locations. This fixes a warning that the cgroup memory observer was disabled because no cgroups v1 or v2 current memory file could be found. [#62903](https://github.com/ClickHouse/ClickHouse/pull/62903) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix subsequent use of external tables in client. [#62964](https://github.com/ClickHouse/ClickHouse/pull/62964) ([Azat Khuzhin](https://github.com/azat)).
+* Fix crash with untuple and unresolved lambda. [#63131](https://github.com/ClickHouse/ClickHouse/pull/63131) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix bug which could lead to server to accept connections before server is actually loaded. [#63181](https://github.com/ClickHouse/ClickHouse/pull/63181) ([alesapin](https://github.com/alesapin)).
+* Fix intersect parts when restart after drop range. [#63202](https://github.com/ClickHouse/ClickHouse/pull/63202) ([Han Fei](https://github.com/hanfei1991)).
+* Fix a misbehavior when SQL security defaults don't load for old tables during server startup. [#63209](https://github.com/ClickHouse/ClickHouse/pull/63209) ([pufit](https://github.com/pufit)).
+* JOIN filter push down filled join fix. Closes [#63228](https://github.com/ClickHouse/ClickHouse/issues/63228). [#63234](https://github.com/ClickHouse/ClickHouse/pull/63234) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fix infinite loop while listing objects in Azure blob storage. [#63257](https://github.com/ClickHouse/ClickHouse/pull/63257) ([Julia Kartseva](https://github.com/jkartseva)).
+* CROSS join can be executed with any value `join_algorithm` setting, close [#62431](https://github.com/ClickHouse/ClickHouse/issues/62431). [#63273](https://github.com/ClickHouse/ClickHouse/pull/63273) ([vdimir](https://github.com/vdimir)).
+* Fixed a potential crash caused by a `no space left` error when temporary data in the cache is used. [#63346](https://github.com/ClickHouse/ClickHouse/pull/63346) ([vdimir](https://github.com/vdimir)).
+* Fix bug which could potentially lead to rare LOGICAL_ERROR during SELECT query with message: `Unexpected return type from materialize. Expected type_XXX. Got type_YYY.` Introduced in [#59379](https://github.com/ClickHouse/ClickHouse/issues/59379). [#63353](https://github.com/ClickHouse/ClickHouse/pull/63353) ([alesapin](https://github.com/alesapin)).
+* Fix `X-ClickHouse-Timezone` header returning wrong timezone when using `session_timezone` as query level setting. [#63377](https://github.com/ClickHouse/ClickHouse/pull/63377) ([Andrey Zvonov](https://github.com/zvonand)).
+* Fix debug assert when using grouping WITH ROLLUP and LowCardinality types. [#63398](https://github.com/ClickHouse/ClickHouse/pull/63398) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix logical errors in queries with `GROUPING SETS` and `WHERE` and `group_by_use_nulls = true`, close [#60538](https://github.com/ClickHouse/ClickHouse/issues/60538). [#63405](https://github.com/ClickHouse/ClickHouse/pull/63405) ([vdimir](https://github.com/vdimir)).
+* Fix backup of projection part in case projection was removed from table metadata, but part still has projection. [#63426](https://github.com/ClickHouse/ClickHouse/pull/63426) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix 'Every derived table must have its own alias' error for MYSQL dictionary source, close [#63341](https://github.com/ClickHouse/ClickHouse/issues/63341). [#63481](https://github.com/ClickHouse/ClickHouse/pull/63481) ([vdimir](https://github.com/vdimir)).
+* Insert QueryFinish on AsyncInsertFlush with no data. [#63483](https://github.com/ClickHouse/ClickHouse/pull/63483) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix `system.query_log.used_dictionaries` logging. [#63487](https://github.com/ClickHouse/ClickHouse/pull/63487) ([Eduard Karacharov](https://github.com/korowa)).
+* Support executing function during assignment of parameterized view value. [#63502](https://github.com/ClickHouse/ClickHouse/pull/63502) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Avoid segafult in `MergeTreePrefetchedReadPool` while fetching projection parts. [#63513](https://github.com/ClickHouse/ClickHouse/pull/63513) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix rabbitmq heap-use-after-free found by clang-18, which can happen if an error is thrown from RabbitMQ during initialization of exchange and queues. [#63515](https://github.com/ClickHouse/ClickHouse/pull/63515) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix crash on exit with sentry enabled (due to openssl destroyed before sentry). [#63548](https://github.com/ClickHouse/ClickHouse/pull/63548) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed parquet memory tracking. [#63584](https://github.com/ClickHouse/ClickHouse/pull/63584) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix support for Array and Map with Keyed hashing functions and materialized keys. [#63628](https://github.com/ClickHouse/ClickHouse/pull/63628) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Fixed Parquet filter pushdown not working with Analyzer. [#63642](https://github.com/ClickHouse/ClickHouse/pull/63642) ([Michael Kolupaev](https://github.com/al13n321)).
+* It is forbidden to convert MergeTree to replicated if the zookeeper path for this table already exists. [#63670](https://github.com/ClickHouse/ClickHouse/pull/63670) ([Kirill](https://github.com/kirillgarbar)).
+* Read only the necessary columns from VIEW (new analyzer). Closes [#62594](https://github.com/ClickHouse/ClickHouse/issues/62594). [#63688](https://github.com/ClickHouse/ClickHouse/pull/63688) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fix rare case with missing data in the result of distributed query. [#63691](https://github.com/ClickHouse/ClickHouse/pull/63691) ([vdimir](https://github.com/vdimir)).
+* Fix [#63539](https://github.com/ClickHouse/ClickHouse/issues/63539). Forbid WINDOW redefinition in new analyzer. [#63694](https://github.com/ClickHouse/ClickHouse/pull/63694) ([Dmitry Novik](https://github.com/novikd)).
+* Flatten_nested is broken with replicated database. [#63695](https://github.com/ClickHouse/ClickHouse/pull/63695) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix `SIZES_OF_COLUMNS_DOESNT_MATCH` error for queries with `arrayJoin` function in `WHERE`. Fixes [#63653](https://github.com/ClickHouse/ClickHouse/issues/63653). [#63722](https://github.com/ClickHouse/ClickHouse/pull/63722) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix `Not found column` and `CAST AS Map from array requires nested tuple of 2 elements` exceptions for distributed queries which use `Map(Nothing, Nothing)` type. Fixes [#63637](https://github.com/ClickHouse/ClickHouse/issues/63637). [#63753](https://github.com/ClickHouse/ClickHouse/pull/63753) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix possible `ILLEGAL_COLUMN` error in `partial_merge` join, close [#37928](https://github.com/ClickHouse/ClickHouse/issues/37928). [#63755](https://github.com/ClickHouse/ClickHouse/pull/63755) ([vdimir](https://github.com/vdimir)).
+* `query_plan_remove_redundant_distinct` can break queries with WINDOW FUNCTIONS (with `allow_experimental_analyzer` is on). Fixes [#62820](https://github.com/ClickHouse/ClickHouse/issues/62820). [#63776](https://github.com/ClickHouse/ClickHouse/pull/63776) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix possible crash with SYSTEM UNLOAD PRIMARY KEY. [#63778](https://github.com/ClickHouse/ClickHouse/pull/63778) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix a query with a duplicating cycling alias. Fixes [#63320](https://github.com/ClickHouse/ClickHouse/issues/63320). [#63791](https://github.com/ClickHouse/ClickHouse/pull/63791) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed performance degradation of parsing data formats in INSERT query. This closes [#62918](https://github.com/ClickHouse/ClickHouse/issues/62918). This partially reverts [#42284](https://github.com/ClickHouse/ClickHouse/issues/42284), which breaks the original design and introduces more problems. [#63801](https://github.com/ClickHouse/ClickHouse/pull/63801) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add 'endpoint_subpath' S3 URI setting to allow plain_rewritable disks to share the same endpoint. [#63806](https://github.com/ClickHouse/ClickHouse/pull/63806) ([Julia Kartseva](https://github.com/jkartseva)).
+* Fix queries using parallel read buffer (e.g. with max_download_thread > 0) getting stuck when threads cannot be allocated. [#63814](https://github.com/ClickHouse/ClickHouse/pull/63814) ([Antonio Andelic](https://github.com/antonio2368)).
+* Allow JOIN filter push down to both streams if only single equivalent column is used in query. Closes [#63799](https://github.com/ClickHouse/ClickHouse/issues/63799). [#63819](https://github.com/ClickHouse/ClickHouse/pull/63819) ([Maksim Kita](https://github.com/kitaisreal)).
+* Remove the data from all disks after DROP with the Lazy database engines. Without these changes, orhpaned will remain on the disks. [#63848](https://github.com/ClickHouse/ClickHouse/pull/63848) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Fix incorrect select query result when parallel replicas were used to read from a Materialized View. [#63861](https://github.com/ClickHouse/ClickHouse/pull/63861) ([Nikita Taranov](https://github.com/nickitat)).
+* Fixes in `find_super_nodes` and `find_big_family` command of keeper-client: - do not fail on ZNONODE errors - find super nodes inside super nodes - properly calculate subtree node count. [#63862](https://github.com/ClickHouse/ClickHouse/pull/63862) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix a error `Database name is empty` for remote queries with lambdas over the cluster with modified default database. Fixes [#63471](https://github.com/ClickHouse/ClickHouse/issues/63471). [#63864](https://github.com/ClickHouse/ClickHouse/pull/63864) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix SIGSEGV due to CPU/Real (`query_profiler_real_time_period_ns`/`query_profiler_cpu_time_period_ns`) profiler (has been an issue since 2022, that leads to periodic server crashes, especially if you were using distributed engine). [#63865](https://github.com/ClickHouse/ClickHouse/pull/63865) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed `EXPLAIN CURRENT TRANSACTION` query. [#63926](https://github.com/ClickHouse/ClickHouse/pull/63926) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix analyzer - IN function with arbitrary deep sub-selects in materialized view to use insertion block. [#63930](https://github.com/ClickHouse/ClickHouse/pull/63930) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Allow `ALTER TABLE .. MODIFY|RESET SETTING` and `ALTER TABLE .. MODIFY COMMENT` for plain_rewritable disk. [#63933](https://github.com/ClickHouse/ClickHouse/pull/63933) ([Julia Kartseva](https://github.com/jkartseva)).
+* Fix Recursive CTE with distributed queries. Closes [#63790](https://github.com/ClickHouse/ClickHouse/issues/63790). [#63939](https://github.com/ClickHouse/ClickHouse/pull/63939) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fixed reading of columns of type `Tuple(Map(LowCardinality(String), String), ...)`. [#63956](https://github.com/ClickHouse/ClickHouse/pull/63956) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix resolve of unqualified COLUMNS matcher. Preserve the input columns order and forbid usage of unknown identifiers. [#63962](https://github.com/ClickHouse/ClickHouse/pull/63962) ([Dmitry Novik](https://github.com/novikd)).
+* Fix the `Not found column` error for queries with `skip_unused_shards = 1`, `LIMIT BY`, and the new analyzer. Fixes [#63943](https://github.com/ClickHouse/ClickHouse/issues/63943). [#63983](https://github.com/ClickHouse/ClickHouse/pull/63983) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* (Low-quality third-party Kusto Query Language). Resolve Client Abortion Issue When Using KQL Table Function in Interactive Mode. [#63992](https://github.com/ClickHouse/ClickHouse/pull/63992) ([Yong Wang](https://github.com/kashwy)).
+* Fix an `Cyclic aliases` error for cyclic aliases of different type (expression and function). [#63993](https://github.com/ClickHouse/ClickHouse/pull/63993) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Deserialize untrusted binary inputs in a safer way. [#64024](https://github.com/ClickHouse/ClickHouse/pull/64024) ([Robert Schulze](https://github.com/rschu1ze)).
+* Do not throw `Storage doesn't support FINAL` error for remote queries over non-MergeTree tables with `final = true` and new analyzer. Fixes [#63960](https://github.com/ClickHouse/ClickHouse/issues/63960). [#64037](https://github.com/ClickHouse/ClickHouse/pull/64037) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Add missing settings to recoverLostReplica. [#64040](https://github.com/ClickHouse/ClickHouse/pull/64040) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix unwind on SIGSEGV on aarch64 (due to small stack for signal). [#64058](https://github.com/ClickHouse/ClickHouse/pull/64058) ([Azat Khuzhin](https://github.com/azat)).
+* This fix will use a proper redefined context with the correct definer for each individual view in the query pipeline. [#64079](https://github.com/ClickHouse/ClickHouse/pull/64079) ([pufit](https://github.com/pufit)).
+* Fix analyzer: "Not found column" error is fixed when using INTERPOLATE. [#64096](https://github.com/ClickHouse/ClickHouse/pull/64096) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix azure backup writing multipart blocks as 1mb (read buffer size) instead of max_upload_part_size. [#64117](https://github.com/ClickHouse/ClickHouse/pull/64117) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix creating backups to S3 buckets with different credentials from the disk containing the file. [#64153](https://github.com/ClickHouse/ClickHouse/pull/64153) ([Antonio Andelic](https://github.com/antonio2368)).
+* Prevent LOGICAL_ERROR on CREATE TABLE as MaterializedView. [#64174](https://github.com/ClickHouse/ClickHouse/pull/64174) ([Raúl Marín](https://github.com/Algunenano)).
+* The query cache now considers two identical queries against different databases as different. The previous behavior could be used to bypass missing privileges to read from a table. [#64199](https://github.com/ClickHouse/ClickHouse/pull/64199) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix possible abort on uncaught exception in ~WriteBufferFromFileDescriptor in StatusFile. [#64206](https://github.com/ClickHouse/ClickHouse/pull/64206) ([Kruglov Pavel](https://github.com/Avogar)).
+* Ignore `text_log` config when using Keeper. [#64218](https://github.com/ClickHouse/ClickHouse/pull/64218) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix `duplicate alias` error for distributed queries with `ARRAY JOIN`. [#64226](https://github.com/ClickHouse/ClickHouse/pull/64226) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix unexpected accurateCast from string to integer. [#64255](https://github.com/ClickHouse/ClickHouse/pull/64255) ([wudidapaopao](https://github.com/wudidapaopao)).
+* Fixed CNF simplification, in case any OR group contains mutually exclusive atoms. [#64256](https://github.com/ClickHouse/ClickHouse/pull/64256) ([Eduard Karacharov](https://github.com/korowa)).
+* Fix Query Tree size validation. [#64377](https://github.com/ClickHouse/ClickHouse/pull/64377) ([Dmitry Novik](https://github.com/novikd)).
+* Fix `Logical error: Bad cast` for `Buffer` table with `PREWHERE`. [#64388](https://github.com/ClickHouse/ClickHouse/pull/64388) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Prevent recursive logging in `blob_storage_log` when it's stored on object storage. [#64393](https://github.com/ClickHouse/ClickHouse/pull/64393) ([vdimir](https://github.com/vdimir)).
+* Fixed `CREATE TABLE AS` queries for tables with default expressions. [#64455](https://github.com/ClickHouse/ClickHouse/pull/64455) ([Anton Popov](https://github.com/CurtizJ)).
+* Fixed `optimize_read_in_order` behaviour for ORDER BY ... NULLS FIRST / LAST on tables with nullable keys. [#64483](https://github.com/ClickHouse/ClickHouse/pull/64483) ([Eduard Karacharov](https://github.com/korowa)).
+* Fix the `Expression nodes list expected 1 projection names` and `Unknown expression or identifier` errors for queries with aliases to `GLOBAL IN.`. [#64517](https://github.com/ClickHouse/ClickHouse/pull/64517) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix an error `Cannot find column` in distributed queries with constant CTE in the `GROUP BY` key. [#64519](https://github.com/ClickHouse/ClickHouse/pull/64519) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed ORC statistics calculation, when writing, for unsigned types on all platforms and Int8 on ARM. [#64563](https://github.com/ClickHouse/ClickHouse/pull/64563) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix the crash loop when restoring from backup is blocked by creating an MV with a definer that hasn't been restored yet. [#64595](https://github.com/ClickHouse/ClickHouse/pull/64595) ([pufit](https://github.com/pufit)).
+* Fix the output of function `formatDateTimeInJodaSyntax` when a formatter generates an uneven number of characters and the last character is `0`. For example, `SELECT formatDateTimeInJodaSyntax(toDate('2012-05-29'), 'D')` now correctly returns `150` instead of previously `15`. [#64614](https://github.com/ClickHouse/ClickHouse/pull/64614) ([LiuNeng](https://github.com/liuneng1994)).
+* Do not rewrite aggregation if `-If` combinator is already used. [#64638](https://github.com/ClickHouse/ClickHouse/pull/64638) ([Dmitry Novik](https://github.com/novikd)).
+* Fix type inference for float (in case of small buffer, i.e. `--max_read_buffer_size 1`). [#64641](https://github.com/ClickHouse/ClickHouse/pull/64641) ([Azat Khuzhin](https://github.com/azat)).
+* Fix bug which could lead to non-working TTLs with expressions. [#64694](https://github.com/ClickHouse/ClickHouse/pull/64694) ([alesapin](https://github.com/alesapin)).
+* Fix removing the `WHERE` and `PREWHERE` expressions, which are always true (for the new analyzer). [#64695](https://github.com/ClickHouse/ClickHouse/pull/64695) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed excessive part elimination by token-based text indexes (`ngrambf` , `full_text`) when filtering by result of `startsWith`, `endsWith`, `match`, `multiSearchAny`. [#64720](https://github.com/ClickHouse/ClickHouse/pull/64720) ([Eduard Karacharov](https://github.com/korowa)).
+* Fixes incorrect behaviour of ANSI CSI escaping in the `UTF8::computeWidth` function. [#64756](https://github.com/ClickHouse/ClickHouse/pull/64756) ([Shaun Struwig](https://github.com/Blargian)).
+* Fix a case of incorrect removal of `ORDER BY` / `LIMIT BY` across subqueries. [#64766](https://github.com/ClickHouse/ClickHouse/pull/64766) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix (experimental) unequal join with subqueries for sets which are in the mixed join conditions. [#64775](https://github.com/ClickHouse/ClickHouse/pull/64775) ([lgbo](https://github.com/lgbo-ustc)).
+* Fix crash in a local cache over `plain_rewritable` disk. [#64778](https://github.com/ClickHouse/ClickHouse/pull/64778) ([Julia Kartseva](https://github.com/jkartseva)).
+* Keeper fix: return correct value for `zk_latest_snapshot_size` in `mntr` command. [#64784](https://github.com/ClickHouse/ClickHouse/pull/64784) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix `Cannot find column` in distributed query with `ARRAY JOIN` by `Nested` column. Fixes [#64755](https://github.com/ClickHouse/ClickHouse/issues/64755). [#64801](https://github.com/ClickHouse/ClickHouse/pull/64801) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix memory leak in slru cache policy. [#64803](https://github.com/ClickHouse/ClickHouse/pull/64803) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fixed possible incorrect memory tracking in several kinds of queries: queries that read any data from S3, queries via http protocol, asynchronous inserts. [#64844](https://github.com/ClickHouse/ClickHouse/pull/64844) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix the `Block structure mismatch` error for queries reading with `PREWHERE` from the materialized view when the materialized view has columns of different types than the source table. Fixes [#64611](https://github.com/ClickHouse/ClickHouse/issues/64611). [#64855](https://github.com/ClickHouse/ClickHouse/pull/64855) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix rare crash when table has TTL with subquery + database replicated + parallel replicas + analyzer. It's really rare, but please don't use TTLs with subqueries. [#64858](https://github.com/ClickHouse/ClickHouse/pull/64858) ([alesapin](https://github.com/alesapin)).
+* Fix duplicating `Delete` events in `blob_storage_log` in case of large batch to delete. [#64924](https://github.com/ClickHouse/ClickHouse/pull/64924) ([vdimir](https://github.com/vdimir)).
+* Backported in [#65544](https://github.com/ClickHouse/ClickHouse/issues/65544): Fix crash for `ALTER TABLE ... ON CLUSTER ... MODIFY SQL SECURITY`. [#64957](https://github.com/ClickHouse/ClickHouse/pull/64957) ([pufit](https://github.com/pufit)).
+* Fixed `Session moved to another server` error from [Zoo]Keeper that might happen after server startup when the config has includes from [Zoo]Keeper. [#64986](https://github.com/ClickHouse/ClickHouse/pull/64986) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#65582](https://github.com/ClickHouse/ClickHouse/issues/65582): Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix `ALTER MODIFY COMMENT` query that was broken for parameterized VIEWs in https://github.com/ClickHouse/ClickHouse/pull/54211. [#65031](https://github.com/ClickHouse/ClickHouse/pull/65031) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix `host_id` in DatabaseReplicated when `cluster_secure_connection` parameter is enabled. Previously all the connections within the cluster created by DatabaseReplicated were not secure, even if the parameter was enabled. [#65054](https://github.com/ClickHouse/ClickHouse/pull/65054) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fixing the `Not-ready Set` error after the `PREWHERE` optimization for StorageMerge. [#65057](https://github.com/ClickHouse/ClickHouse/pull/65057) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Avoid writing to finalized buffer in File-like storages. [#65063](https://github.com/ClickHouse/ClickHouse/pull/65063) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix possible infinite query duration in case of cyclic aliases. Fixes [#64849](https://github.com/ClickHouse/ClickHouse/issues/64849). [#65081](https://github.com/ClickHouse/ClickHouse/pull/65081) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix the `Unknown expression identifier` error for remote queries with `INTERPOLATE (alias)` (new analyzer). Fixes [#64636](https://github.com/ClickHouse/ClickHouse/issues/64636). [#65090](https://github.com/ClickHouse/ClickHouse/pull/65090) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix pushing arithmetic operations out of aggregation. In the new analyzer, optimization was applied only once. [#65104](https://github.com/ClickHouse/ClickHouse/pull/65104) ([Dmitry Novik](https://github.com/novikd)).
+* Fix aggregate function name rewriting in the new analyzer. [#65110](https://github.com/ClickHouse/ClickHouse/pull/65110) ([Dmitry Novik](https://github.com/novikd)).
+* Respond with 5xx instead of 200 OK in case of receive timeout while reading (parts of) the request body from the client socket. [#65118](https://github.com/ClickHouse/ClickHouse/pull/65118) ([Julian Maicher](https://github.com/jmaicher)).
+* Backported in [#65734](https://github.com/ClickHouse/ClickHouse/issues/65734): Eliminate injective function in argument of functions `uniq*` recursively. This used to work correctly but was broken in the new analyzer. [#65140](https://github.com/ClickHouse/ClickHouse/pull/65140) ([Duc Canh Le](https://github.com/canhld94)).
+* Fix possible crash for hedged requests. [#65206](https://github.com/ClickHouse/ClickHouse/pull/65206) ([Azat Khuzhin](https://github.com/azat)).
+* Fix the bug in Hashed and Hashed_Array dictionary short circuit evaluation, which may read uninitialized number, leading to various errors. [#65256](https://github.com/ClickHouse/ClickHouse/pull/65256) ([jsc0218](https://github.com/jsc0218)).
+* This PR ensures that the type of the constant(IN operator's second parameter) is always visible during the IN operator's type conversion process. Otherwise, losing type information may cause some conversions to fail, such as the conversion from DateTime to Date. fix ([#64487](https://github.com/ClickHouse/ClickHouse/issues/64487)). [#65315](https://github.com/ClickHouse/ClickHouse/pull/65315) ([pn](https://github.com/chloro-pn)).
+* Backported in [#65665](https://github.com/ClickHouse/ClickHouse/issues/65665): Disable `non-intersecting-parts` optimization for queries with `FINAL` in case of `read-in-order` optimization was enabled. This could lead to an incorrect query result. As a workaround, disable `do_not_merge_across_partitions_select_final` and `split_parts_ranges_into_intersecting_and_non_intersecting_final` before this fix is merged. [#65505](https://github.com/ClickHouse/ClickHouse/pull/65505) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65606](https://github.com/ClickHouse/ClickHouse/issues/65606): Fix getting exception `Index out of bound for blob metadata` in case all files from list batch were filtered out. [#65523](https://github.com/ClickHouse/ClickHouse/pull/65523) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#65790](https://github.com/ClickHouse/ClickHouse/issues/65790): Fixed bug in MergeJoin. Column in sparse serialisation might be treated as a column of its nested type though the required conversion wasn't performed. [#65632](https://github.com/ClickHouse/ClickHouse/pull/65632) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#65814](https://github.com/ClickHouse/ClickHouse/issues/65814): Fix invalid exceptions in function `parseDateTime` with `%F` and `%D` placeholders. [#65768](https://github.com/ClickHouse/ClickHouse/pull/65768) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#65830](https://github.com/ClickHouse/ClickHouse/issues/65830): Fix a bug in short circuit logic when old analyzer and dictGetOrDefault is used. [#65802](https://github.com/ClickHouse/ClickHouse/pull/65802) ([jsc0218](https://github.com/jsc0218)).
+
+#### Build/Testing/Packaging Improvement
+* ClickHouse is built with clang-18. A lot of new checks from clang-tidy-18 have been enabled. [#60469](https://github.com/ClickHouse/ClickHouse/pull/60469) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Make `network` service be required when using the rc init script to start the ClickHouse server daemon. [#60650](https://github.com/ClickHouse/ClickHouse/pull/60650) ([Chun-Sheng, Li](https://github.com/peter279k)).
+* Re-enable broken s390x build in CI. [#63135](https://github.com/ClickHouse/ClickHouse/pull/63135) ([Harry Lee](https://github.com/HarryLeeIBM)).
+* The Dockerfile is reviewed by the docker official library in https://github.com/docker-library/official-images/pull/15846. [#63400](https://github.com/ClickHouse/ClickHouse/pull/63400) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Information about every symbol in every translation unit will be collected in the CI database for every build in the CI. This closes [#63494](https://github.com/ClickHouse/ClickHouse/issues/63494). [#63495](https://github.com/ClickHouse/ClickHouse/pull/63495) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Experimentally support loongarch64 as a new platform for ClickHouse. [#63733](https://github.com/ClickHouse/ClickHouse/pull/63733) ([qiangxuhui](https://github.com/qiangxuhui)).
+* Update Apache Datasketches library. It resolves [#63858](https://github.com/ClickHouse/ClickHouse/issues/63858). [#63923](https://github.com/ClickHouse/ClickHouse/pull/63923) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Enable GRPC support for aarch64 linux while cross-compiling binary. [#64072](https://github.com/ClickHouse/ClickHouse/pull/64072) ([alesapin](https://github.com/alesapin)).
+* Fix typo in test_hdfsCluster_unset_skip_unavailable_shards. The test writes data to unskip_unavailable_shards, but uses skip_unavailable_shards from the previous test. [#64243](https://github.com/ClickHouse/ClickHouse/pull/64243) ([Mikhail Artemenko](https://github.com/Michicosun)).
+* Reduce the size of some slow tests. [#64387](https://github.com/ClickHouse/ClickHouse/pull/64387) ([Raúl Marín](https://github.com/Algunenano)).
+* Reduce the size of some slow tests. [#64452](https://github.com/ClickHouse/ClickHouse/pull/64452) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix test_lost_part_other_replica. [#64512](https://github.com/ClickHouse/ClickHouse/pull/64512) ([Raúl Marín](https://github.com/Algunenano)).
+* Add tests for experimental unequal joins and randomize new settings in clickhouse-test. [#64535](https://github.com/ClickHouse/ClickHouse/pull/64535) ([Nikita Fomichev](https://github.com/fm4v)).
+* Upgrade tests: Update config and work with release candidates. [#64542](https://github.com/ClickHouse/ClickHouse/pull/64542) ([Raúl Marín](https://github.com/Algunenano)).
+* Add support for LLVM XRay. [#64592](https://github.com/ClickHouse/ClickHouse/pull/64592) ([Tomer Shafir](https://github.com/tomershafir)).
+* Speed up 02995_forget_partition. [#64761](https://github.com/ClickHouse/ClickHouse/pull/64761) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix 02790_async_queries_in_query_log. [#64764](https://github.com/ClickHouse/ClickHouse/pull/64764) ([Raúl Marín](https://github.com/Algunenano)).
+* Support LLVM XRay on Linux amd64 only. [#64837](https://github.com/ClickHouse/ClickHouse/pull/64837) ([Tomer Shafir](https://github.com/tomershafir)).
+* Get rid of custom code in `tests/ci/download_release_packages.py` and `tests/ci/get_previous_release_tag.py` to avoid issues after the https://github.com/ClickHouse/ClickHouse/pull/64759 is merged. [#64848](https://github.com/ClickHouse/ClickHouse/pull/64848) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Decrease the `unit-test` image a few times. [#65102](https://github.com/ClickHouse/ClickHouse/pull/65102) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### NO CL CATEGORY
+
+* Backported in [#65568](https://github.com/ClickHouse/ClickHouse/issues/65568):. [#65498](https://github.com/ClickHouse/ClickHouse/pull/65498) ([Sergei Trifonov](https://github.com/serxa)).
+* Backported in [#65693](https://github.com/ClickHouse/ClickHouse/issues/65693):. [#65686](https://github.com/ClickHouse/ClickHouse/pull/65686) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### NO CL ENTRY
+
+* NO CL ENTRY: 'Revert "Do not remove server constants from GROUP BY key for secondary query."'. [#63297](https://github.com/ClickHouse/ClickHouse/pull/63297) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* NO CL ENTRY: 'Revert "Introduce bulk loading to StorageEmbeddedRocksDB"'. [#63316](https://github.com/ClickHouse/ClickHouse/pull/63316) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* NO CL ENTRY: 'Revert "Revert "Do not remove server constants from GROUP BY key for secondary query.""'. [#63415](https://github.com/ClickHouse/ClickHouse/pull/63415) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* NO CL ENTRY: 'Revert "Fix index analysis for `DateTime64`"'. [#63525](https://github.com/ClickHouse/ClickHouse/pull/63525) ([Raúl Marín](https://github.com/Algunenano)).
+* NO CL ENTRY: 'Revert "Update gui.md - Add ch-ui to open-source available tools."'. [#64064](https://github.com/ClickHouse/ClickHouse/pull/64064) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* NO CL ENTRY: 'Revert "Prevent conversion to Replicated if zookeeper path already exists"'. [#64214](https://github.com/ClickHouse/ClickHouse/pull/64214) ([Sergei Trifonov](https://github.com/serxa)).
+* NO CL ENTRY: 'Revert "Refactoring of Server.h: Isolate server management from other logic"'. [#64425](https://github.com/ClickHouse/ClickHouse/pull/64425) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* NO CL ENTRY: 'Revert "Remove some unnecessary `UNREACHABLE`s"'. [#64430](https://github.com/ClickHouse/ClickHouse/pull/64430) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* NO CL ENTRY: 'Revert "CI: fix build_report selection in case of job reuse"'. [#64516](https://github.com/ClickHouse/ClickHouse/pull/64516) ([Max K.](https://github.com/maxknv)).
+* NO CL ENTRY: 'Revert "Revert "CI: fix build_report selection in case of job reuse""'. [#64531](https://github.com/ClickHouse/ClickHouse/pull/64531) ([Max K.](https://github.com/maxknv)).
+* NO CL ENTRY: 'Revert "Add `fromReadableSize` function"'. [#64616](https://github.com/ClickHouse/ClickHouse/pull/64616) ([Robert Schulze](https://github.com/rschu1ze)).
+* NO CL ENTRY: 'Update CHANGELOG.md'. [#64816](https://github.com/ClickHouse/ClickHouse/pull/64816) ([Paweł Kudzia](https://github.com/pakud)).
+* NO CL ENTRY: 'Revert "Reduce lock contention for MergeTree tables (by renaming parts without holding lock)"'. [#64899](https://github.com/ClickHouse/ClickHouse/pull/64899) ([alesapin](https://github.com/alesapin)).
+* NO CL ENTRY: 'Revert "Add dynamic untracked memory limits for more precise memory tracking"'. [#64969](https://github.com/ClickHouse/ClickHouse/pull/64969) ([Sergei Trifonov](https://github.com/serxa)).
+* NO CL ENTRY: 'Revert "Fix duplicating Delete events in blob_storage_log"'. [#65049](https://github.com/ClickHouse/ClickHouse/pull/65049) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* NO CL ENTRY: 'Revert "Revert "Fix duplicating Delete events in blob_storage_log""'. [#65053](https://github.com/ClickHouse/ClickHouse/pull/65053) ([vdimir](https://github.com/vdimir)).
+* NO CL ENTRY: 'Revert "S3: reduce retires time for queries, increase retries count for backups"'. [#65148](https://github.com/ClickHouse/ClickHouse/pull/65148) ([Raúl Marín](https://github.com/Algunenano)).
+* NO CL ENTRY: 'Revert "Small fix for 02340_parts_refcnt_mergetree"'. [#65149](https://github.com/ClickHouse/ClickHouse/pull/65149) ([Raúl Marín](https://github.com/Algunenano)).
+* NO CL ENTRY: 'Revert "Change default s3_throw_on_zero_files_match to true, document that presigned S3 URLs are not supported"'. [#65250](https://github.com/ClickHouse/ClickHouse/pull/65250) ([Max K.](https://github.com/maxknv)).
+* NO CL ENTRY: 'Revert "Fix AWS ECS"'. [#65361](https://github.com/ClickHouse/ClickHouse/pull/65361) ([Alexander Tokmakov](https://github.com/tavplubix)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Try abort on current thread join. [#42544](https://github.com/ClickHouse/ClickHouse/pull/42544) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* This change was reverted. [#51008](https://github.com/ClickHouse/ClickHouse/pull/51008) ([Michael Kolupaev](https://github.com/al13n321)).
+* Analyzer fuzzer 2. [#57098](https://github.com/ClickHouse/ClickHouse/pull/57098) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Analyzer fuzzer 4. [#57101](https://github.com/ClickHouse/ClickHouse/pull/57101) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Check python code with flake8. [#58349](https://github.com/ClickHouse/ClickHouse/pull/58349) ([Azat Khuzhin](https://github.com/azat)).
+* Unite s3/hdfs/azure storage implementations into a single class working with IObjectStorage. Same for *Cluster, data lakes and Queue storages. [#59767](https://github.com/ClickHouse/ClickHouse/pull/59767) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Remove http_max_chunk_size setting (too internal). [#60852](https://github.com/ClickHouse/ClickHouse/pull/60852) ([Azat Khuzhin](https://github.com/azat)).
+* Fix race in refreshable materialized views causing SELECT to fail sometimes. [#60883](https://github.com/ClickHouse/ClickHouse/pull/60883) ([Michael Kolupaev](https://github.com/al13n321)).
+* Refactor KeyCondition and key analysis to improve PartitionPruner and trivial count optimization. This is separated from [#60463](https://github.com/ClickHouse/ClickHouse/issues/60463) . [#61459](https://github.com/ClickHouse/ClickHouse/pull/61459) ([Amos Bird](https://github.com/amosbird)).
+* Implement cumulative A Sync status. [#61464](https://github.com/ClickHouse/ClickHouse/pull/61464) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Parallel replicas: table check failover. [#61935](https://github.com/ClickHouse/ClickHouse/pull/61935) ([Igor Nikonov](https://github.com/devcrafter)).
+* This change was reverted. [#61973](https://github.com/ClickHouse/ClickHouse/pull/61973) ([Azat Khuzhin](https://github.com/azat)).
+* Avoid crashing on column type mismatch in a few dozen places. [#62087](https://github.com/ClickHouse/ClickHouse/pull/62087) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix optimize_if_chain_to_multiif const NULL handling. [#62104](https://github.com/ClickHouse/ClickHouse/pull/62104) ([Michael Kolupaev](https://github.com/al13n321)).
+* Use intrusive lists for `ResourceRequest` instead of deque. [#62165](https://github.com/ClickHouse/ClickHouse/pull/62165) ([Sergei Trifonov](https://github.com/serxa)).
+* Analyzer: Fix validateAggregates for tables with different aliases. [#62346](https://github.com/ClickHouse/ClickHouse/pull/62346) ([vdimir](https://github.com/vdimir)).
+* Improve code and tests of `DROP` of multiple tables. [#62359](https://github.com/ClickHouse/ClickHouse/pull/62359) ([zhongyuankai](https://github.com/zhongyuankai)).
+* Fix exception message during writing to partitioned s3/hdfs/azure path with globs. [#62423](https://github.com/ClickHouse/ClickHouse/pull/62423) ([Kruglov Pavel](https://github.com/Avogar)).
+* Support UBSan on Clang-19 (master). [#62466](https://github.com/ClickHouse/ClickHouse/pull/62466) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Replay ZK logs using keeper-bench. [#62481](https://github.com/ClickHouse/ClickHouse/pull/62481) ([Antonio Andelic](https://github.com/antonio2368)).
+* Save the stacktrace of thread waiting on failing AsyncLoader job. [#62719](https://github.com/ClickHouse/ClickHouse/pull/62719) ([Sergei Trifonov](https://github.com/serxa)).
+* group_by_use_nulls strikes back. [#62922](https://github.com/ClickHouse/ClickHouse/pull/62922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Analyzer: prefer column name to alias from array join. [#62995](https://github.com/ClickHouse/ClickHouse/pull/62995) ([vdimir](https://github.com/vdimir)).
+* CI: try separate the workflows file for GitHub's Merge Queue. [#63123](https://github.com/ClickHouse/ClickHouse/pull/63123) ([Max K.](https://github.com/maxknv)).
+* Try to fix coverage tests. [#63130](https://github.com/ClickHouse/ClickHouse/pull/63130) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix azure backup flaky test. [#63158](https://github.com/ClickHouse/ClickHouse/pull/63158) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Merging [#60920](https://github.com/ClickHouse/ClickHouse/issues/60920). [#63159](https://github.com/ClickHouse/ClickHouse/pull/63159) ([vdimir](https://github.com/vdimir)).
+* QueryAnalysisPass improve QUALIFY validation. [#63162](https://github.com/ClickHouse/ClickHouse/pull/63162) ([Maksim Kita](https://github.com/kitaisreal)).
+* Add numpy tests for different endianness. [#63189](https://github.com/ClickHouse/ClickHouse/pull/63189) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Clean the `_work` directory between runner's launches. Fallback to auto-update actions runner if it fails to start. Make the `init-network.sh` sourceable and executable. [#63195](https://github.com/ClickHouse/ClickHouse/pull/63195) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Add ability to run Azure tests in PR with label. [#63196](https://github.com/ClickHouse/ClickHouse/pull/63196) ([alesapin](https://github.com/alesapin)).
+* Fix possible endless loop while reading from azure. [#63197](https://github.com/ClickHouse/ClickHouse/pull/63197) ([Anton Popov](https://github.com/CurtizJ)).
+* Add information about materialized view security bug fix into the changelog. [#63204](https://github.com/ClickHouse/ClickHouse/pull/63204) ([pufit](https://github.com/pufit)).
+* Disable one test from 02994_sanity_check_settings. [#63208](https://github.com/ClickHouse/ClickHouse/pull/63208) ([Raúl Marín](https://github.com/Algunenano)).
+* Enable custom parquet encoder by default, attempt 2. [#63210](https://github.com/ClickHouse/ClickHouse/pull/63210) ([Michael Kolupaev](https://github.com/al13n321)).
+* Update version after release. [#63215](https://github.com/ClickHouse/ClickHouse/pull/63215) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Update version_date.tsv and changelogs after v24.4.1.2088-stable. [#63217](https://github.com/ClickHouse/ClickHouse/pull/63217) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelogs after v24.3.3.102-lts. [#63226](https://github.com/ClickHouse/ClickHouse/pull/63226) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelogs after v24.2.3.70-stable. [#63227](https://github.com/ClickHouse/ClickHouse/pull/63227) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Return back [#61551](https://github.com/ClickHouse/ClickHouse/issues/61551) (More optimal loading of marks). [#63233](https://github.com/ClickHouse/ClickHouse/pull/63233) ([Anton Popov](https://github.com/CurtizJ)).
+* Hide CI options under a spoiler. [#63237](https://github.com/ClickHouse/ClickHouse/pull/63237) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Add azure run with msan. [#63238](https://github.com/ClickHouse/ClickHouse/pull/63238) ([alesapin](https://github.com/alesapin)).
+* Now syntax for this command is following: `TRUNCATE ALL TABLES FROM [IF EXISTS] `. [#63241](https://github.com/ClickHouse/ClickHouse/pull/63241) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Minor follow-up to a renaming PR. [#63260](https://github.com/ClickHouse/ClickHouse/pull/63260) ([Robert Schulze](https://github.com/rschu1ze)).
+* Followup for [#62613](https://github.com/ClickHouse/ClickHouse/issues/62613) Adding back checks similar to these: https://github.com/ClickHouse/ClickHouse/pull/62613/files#diff-70859078da57ecdfc66d26f732c0d7718d269e82bdc80e62b39f5ffeab36c05bL99 https://github.com/ClickHouse/ClickHouse/pull/62613/files#diff-70859078da57ecdfc66d26f732c0d7718d269e82bdc80e62b39f5ffeab36c05bL144-L149. [#63274](https://github.com/ClickHouse/ClickHouse/pull/63274) ([Alexander Gololobov](https://github.com/davenger)).
+* This setting was added in 24.5, not 24.4. [#63278](https://github.com/ClickHouse/ClickHouse/pull/63278) ([Raúl Marín](https://github.com/Algunenano)).
+* Improve cloud backport script. [#63282](https://github.com/ClickHouse/ClickHouse/pull/63282) ([Raúl Marín](https://github.com/Algunenano)).
+* Update version_date.tsv and changelogs after v23.8.14.6-lts. [#63285](https://github.com/ClickHouse/ClickHouse/pull/63285) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix azure flaky test. [#63286](https://github.com/ClickHouse/ClickHouse/pull/63286) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix deadlock in `CacheDictionaryUpdateQueue` in case of exception in constructor. [#63287](https://github.com/ClickHouse/ClickHouse/pull/63287) ([Nikita Taranov](https://github.com/nickitat)).
+* DiskApp: fix 'list --recursive /' and crash on invalid arguments. [#63296](https://github.com/ClickHouse/ClickHouse/pull/63296) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix terminate because of unhandled exception in `MergeTreeDeduplicationLog::shutdown`. [#63298](https://github.com/ClickHouse/ClickHouse/pull/63298) ([Nikita Taranov](https://github.com/nickitat)).
+* Move s3_plain_rewritable unit test to shell. [#63317](https://github.com/ClickHouse/ClickHouse/pull/63317) ([Julia Kartseva](https://github.com/jkartseva)).
+* Add tests for [#63264](https://github.com/ClickHouse/ClickHouse/issues/63264). [#63321](https://github.com/ClickHouse/ClickHouse/pull/63321) ([Raúl Marín](https://github.com/Algunenano)).
+* Try fix segfault in `MergeTreeReadPoolBase::createTask`. [#63323](https://github.com/ClickHouse/ClickHouse/pull/63323) ([Antonio Andelic](https://github.com/antonio2368)).
+* Reduce time-to-insert profiling data in case of logs cluster issues. [#63325](https://github.com/ClickHouse/ClickHouse/pull/63325) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Update README.md. [#63326](https://github.com/ClickHouse/ClickHouse/pull/63326) ([Tyler Hannan](https://github.com/tylerhannan)).
+* This should fix failures with error like `Permission denied ["/var/lib/clickhouse/disks/s3/store/364/3643ff83-0996-4a4a-a90b-a96e66a10c74"]` when table dir was chmod-ed by DatabaseCatalog. [#63330](https://github.com/ClickHouse/ClickHouse/pull/63330) ([Alexander Gololobov](https://github.com/davenger)).
+* Use `/commit/` to have the URLs in [reports](https://play.clickhouse.com/play?user=play#c2VsZWN0IGRpc3RpbmN0IGNvbW1pdF91cmwgZnJvbSBjaGVja3Mgd2hlcmUgY2hlY2tfc3RhcnRfdGltZSA+PSBub3coKSAtIGludGVydmFsIDEgbW9udGggYW5kIHB1bGxfcmVxdWVzdF9udW1iZXI9NjA1MzI=) like https://github.com/ClickHouse/ClickHouse/commit/44f8bc5308b53797bec8cccc3bd29fab8a00235d and not like https://github.com/ClickHouse/ClickHouse/commits/44f8bc5308b53797bec8cccc3bd29fab8a00235d. [#63331](https://github.com/ClickHouse/ClickHouse/pull/63331) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Add test for [#56287](https://github.com/ClickHouse/ClickHouse/issues/56287). [#63340](https://github.com/ClickHouse/ClickHouse/pull/63340) ([Raúl Marín](https://github.com/Algunenano)).
+* Update README.md. [#63350](https://github.com/ClickHouse/ClickHouse/pull/63350) ([Tyler Hannan](https://github.com/tylerhannan)).
+* Add test for [#48049](https://github.com/ClickHouse/ClickHouse/issues/48049). [#63351](https://github.com/ClickHouse/ClickHouse/pull/63351) ([Raúl Marín](https://github.com/Algunenano)).
+* Add option `query_id_prefix` to `clickhouse-benchmark`. [#63352](https://github.com/ClickHouse/ClickHouse/pull/63352) ([Anton Popov](https://github.com/CurtizJ)).
+* New version is fantatish (at least with Ubuntu 22.04.4 LTS): ``` azurite --version /usr/local/lib/node_modules/azurite/dist/src/common/persistence/MemoryExtentStore.js:53 return this._chunks.get(categoryName)?.chunks.get(id); ^. [#63354](https://github.com/ClickHouse/ClickHouse/pull/63354) ([alesapin](https://github.com/alesapin)).
+* Randomize setting `enable_block_offset_column` in stress tests. [#63355](https://github.com/ClickHouse/ClickHouse/pull/63355) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix AST parsing of invalid type names. [#63357](https://github.com/ClickHouse/ClickHouse/pull/63357) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix some 00002_log_and_exception_messages_formatting flakiness. [#63358](https://github.com/ClickHouse/ClickHouse/pull/63358) ([Michael Kolupaev](https://github.com/al13n321)).
+* Add tags for the test 03000_traverse_shadow_system_data_paths.sql to make it stable. [#63366](https://github.com/ClickHouse/ClickHouse/pull/63366) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Add a test for [#55655](https://github.com/ClickHouse/ClickHouse/issues/55655). [#63380](https://github.com/ClickHouse/ClickHouse/pull/63380) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix data race in `reportBrokenPart`. [#63396](https://github.com/ClickHouse/ClickHouse/pull/63396) ([Antonio Andelic](https://github.com/antonio2368)).
+* Workaround for `oklch()` inside canvas bug for firefox. [#63404](https://github.com/ClickHouse/ClickHouse/pull/63404) ([Sergei Trifonov](https://github.com/serxa)).
+* Add test for issue [#47862](https://github.com/ClickHouse/ClickHouse/issues/47862). [#63424](https://github.com/ClickHouse/ClickHouse/pull/63424) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix parsing of `CREATE INDEX` query. [#63425](https://github.com/ClickHouse/ClickHouse/pull/63425) ([Anton Popov](https://github.com/CurtizJ)).
+* We are using Shared Catalog in the CI Logs cluster. [#63442](https://github.com/ClickHouse/ClickHouse/pull/63442) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix collection of coverage data in the CI Logs cluster. [#63453](https://github.com/ClickHouse/ClickHouse/pull/63453) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix flaky test for rocksdb bulk sink. [#63457](https://github.com/ClickHouse/ClickHouse/pull/63457) ([Duc Canh Le](https://github.com/canhld94)).
+* Extra constraints for stress and fuzzer tests. [#63470](https://github.com/ClickHouse/ClickHouse/pull/63470) ([Raúl Marín](https://github.com/Algunenano)).
+* io_uring: refactor get reader from context. [#63475](https://github.com/ClickHouse/ClickHouse/pull/63475) ([Tomer Shafir](https://github.com/tomershafir)).
+* Analyzer setting max_streams_to_max_threads_ratio overflow fix. [#63478](https://github.com/ClickHouse/ClickHouse/pull/63478) ([Maksim Kita](https://github.com/kitaisreal)).
+* Provides setting `output_format_pretty_preserve_border_for_multiline_string` which allows to render multiline strings in pretty format better. The default value for this setting is true. [#63479](https://github.com/ClickHouse/ClickHouse/pull/63479) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fix logical error when reloading config with customly created web disk broken after [#56367](https://github.com/ClickHouse/ClickHouse/issues/56367). [#63484](https://github.com/ClickHouse/ClickHouse/pull/63484) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add test for [#49307](https://github.com/ClickHouse/ClickHouse/issues/49307). [#63486](https://github.com/ClickHouse/ClickHouse/pull/63486) ([Anton Popov](https://github.com/CurtizJ)).
+* Remove leftovers of GCC support in cmake rules. [#63488](https://github.com/ClickHouse/ClickHouse/pull/63488) ([Azat Khuzhin](https://github.com/azat)).
+* Fix ProfileEventTimeIncrement code. [#63489](https://github.com/ClickHouse/ClickHouse/pull/63489) ([Azat Khuzhin](https://github.com/azat)).
+* MergeTreePrefetchedReadPool: Print parent name when logging projection parts. [#63522](https://github.com/ClickHouse/ClickHouse/pull/63522) ([Raúl Marín](https://github.com/Algunenano)).
+* Correctly stop `asyncCopy` tasks in all cases. [#63523](https://github.com/ClickHouse/ClickHouse/pull/63523) ([Antonio Andelic](https://github.com/antonio2368)).
+* Almost everything should work on AArch64 (Part of [#58061](https://github.com/ClickHouse/ClickHouse/issues/58061)). [#63527](https://github.com/ClickHouse/ClickHouse/pull/63527) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Update randomization of `old_parts_lifetime`. [#63530](https://github.com/ClickHouse/ClickHouse/pull/63530) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Update 02240_system_filesystem_cache_table.sh. [#63531](https://github.com/ClickHouse/ClickHouse/pull/63531) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix data race in `DistributedSink`. [#63538](https://github.com/ClickHouse/ClickHouse/pull/63538) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix azure tests run on master. [#63540](https://github.com/ClickHouse/ClickHouse/pull/63540) ([alesapin](https://github.com/alesapin)).
+* The commit 2b8254f987a65d5c21d74fe67b4ee9757970466e was not synced into the cloud because it was falsely marked as a success by `upstream_pr.head.sha`. Here we'll try our best to find a proper commit, and won't make anything if we can't. [#63543](https://github.com/ClickHouse/ClickHouse/pull/63543) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Add `no-s3-storage` tag to local_plain_rewritable ut. [#63546](https://github.com/ClickHouse/ClickHouse/pull/63546) ([Julia Kartseva](https://github.com/jkartseva)).
+* Add `jwcrypto` to integration tests runner. [#63551](https://github.com/ClickHouse/ClickHouse/pull/63551) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Go back to upstream lz4. [#63574](https://github.com/ClickHouse/ClickHouse/pull/63574) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix logical error in ColumnTuple::tryInsert(). [#63583](https://github.com/ClickHouse/ClickHouse/pull/63583) ([Michael Kolupaev](https://github.com/al13n321)).
+* harmonize sumMap error messages on ILLEGAL_TYPE_OF_ARGUMENT. [#63619](https://github.com/ClickHouse/ClickHouse/pull/63619) ([Yohann Jardin](https://github.com/yohannj)).
+* Refactor data part writer to remove dependencies on MergeTreeData and DataPart. [#63620](https://github.com/ClickHouse/ClickHouse/pull/63620) ([Alexander Gololobov](https://github.com/davenger)).
+* Update README.md. [#63631](https://github.com/ClickHouse/ClickHouse/pull/63631) ([Tyler Hannan](https://github.com/tylerhannan)).
+* Ignore global profiler if system.trace_log is not enabled and fix really disable it for keeper standalone build. [#63632](https://github.com/ClickHouse/ClickHouse/pull/63632) ([Azat Khuzhin](https://github.com/azat)).
+* Fixes for 00002_log_and_exception_messages_formatting. [#63634](https://github.com/ClickHouse/ClickHouse/pull/63634) ([Azat Khuzhin](https://github.com/azat)).
+* Fix 02362_part_log_merge_algorithm flaky test. [#63635](https://github.com/ClickHouse/ClickHouse/pull/63635) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
+* Fix tests flakiness due to long SYSTEM FLUSH LOGS (explicitly specify old_parts_lifetime). [#63639](https://github.com/ClickHouse/ClickHouse/pull/63639) ([Azat Khuzhin](https://github.com/azat)).
+* Update clickhouse-test help section. [#63663](https://github.com/ClickHouse/ClickHouse/pull/63663) ([Ali](https://github.com/xogoodnow)).
+* Fix bad test `02950_part_log_bytes_uncompressed`. [#63672](https://github.com/ClickHouse/ClickHouse/pull/63672) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Remove leftovers of `optimize_monotonous_functions_in_order_by`. [#63674](https://github.com/ClickHouse/ClickHouse/pull/63674) ([Nikita Taranov](https://github.com/nickitat)).
+* tests: attempt to fix 02340_parts_refcnt_mergetree flakiness. [#63684](https://github.com/ClickHouse/ClickHouse/pull/63684) ([Azat Khuzhin](https://github.com/azat)).
+* Parallel replicas: simple cleanup. [#63685](https://github.com/ClickHouse/ClickHouse/pull/63685) ([Igor Nikonov](https://github.com/devcrafter)).
+* Cancel S3 reads properly when parallel reads are used. [#63687](https://github.com/ClickHouse/ClickHouse/pull/63687) ([Antonio Andelic](https://github.com/antonio2368)).
+* Explaining insertion order of the Map datatype. [#63690](https://github.com/ClickHouse/ClickHouse/pull/63690) ([Mark Needham](https://github.com/mneedham)).
+* selectRangesToRead() simple cleanup. [#63692](https://github.com/ClickHouse/ClickHouse/pull/63692) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix fuzzed analyzer_join_with_constant query. [#63702](https://github.com/ClickHouse/ClickHouse/pull/63702) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Add missing explicit instantiations of ColumnUnique. [#63718](https://github.com/ClickHouse/ClickHouse/pull/63718) ([Raúl Marín](https://github.com/Algunenano)).
+* Better asserts in ColumnString.h. [#63719](https://github.com/ClickHouse/ClickHouse/pull/63719) ([Raúl Marín](https://github.com/Algunenano)).
+* Try to fix flaky s3 tests test_seekable_formats and test_seekable_formats_url. [#63720](https://github.com/ClickHouse/ClickHouse/pull/63720) ([Kruglov Pavel](https://github.com/Avogar)).
+* Don't randomize some settings in 02941_variant_type_* tests to avoid timeouts. [#63721](https://github.com/ClickHouse/ClickHouse/pull/63721) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix flaky 03145_non_loaded_projection_backup.sh. [#63728](https://github.com/ClickHouse/ClickHouse/pull/63728) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Userspace page cache: don't collect stats if cache is unused. [#63730](https://github.com/ClickHouse/ClickHouse/pull/63730) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix insignificant UBSAN error in QueryAnalyzer::replaceNodesWithPositionalArguments(). [#63734](https://github.com/ClickHouse/ClickHouse/pull/63734) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix a bug in resolving matcher inside lambda inside ARRAY JOIN. [#63744](https://github.com/ClickHouse/ClickHouse/pull/63744) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Self explanatory. [#63754](https://github.com/ClickHouse/ClickHouse/pull/63754) ([Arthur Passos](https://github.com/arthurpassos)).
+* Do not hide disk name. [#63756](https://github.com/ClickHouse/ClickHouse/pull/63756) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* CI: remove Cancel and Debug workflows as redundant. [#63757](https://github.com/ClickHouse/ClickHouse/pull/63757) ([Max K.](https://github.com/maxknv)).
+* Security Policy: Add notification process. [#63773](https://github.com/ClickHouse/ClickHouse/pull/63773) ([Leticia Webb](https://github.com/leticiawebb)).
+* Fix typo. [#63774](https://github.com/ClickHouse/ClickHouse/pull/63774) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix fuzzer when only explicit faults are used. [#63775](https://github.com/ClickHouse/ClickHouse/pull/63775) ([Raúl Marín](https://github.com/Algunenano)).
+* Settings typo. [#63782](https://github.com/ClickHouse/ClickHouse/pull/63782) ([Rory Crispin](https://github.com/RoryCrispin)).
+* Ref. [#63479](https://github.com/ClickHouse/ClickHouse/issues/63479). [#63783](https://github.com/ClickHouse/ClickHouse/pull/63783) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fix test_odbc_interaction from aarch64 [#61457](https://github.com/ClickHouse/ClickHouse/issues/61457). [#63787](https://github.com/ClickHouse/ClickHouse/pull/63787) ([alesapin](https://github.com/alesapin)).
+* Fix test `test_catboost_evaluate` for aarch64. [#61457](https://github.com/ClickHouse/ClickHouse/issues/61457). [#63789](https://github.com/ClickHouse/ClickHouse/pull/63789) ([alesapin](https://github.com/alesapin)).
+* Rewrite plan for parallel replicas in Planner. [#63796](https://github.com/ClickHouse/ClickHouse/pull/63796) ([Igor Nikonov](https://github.com/devcrafter)).
+* Follow-up for the `binary_symbols` table in CI. [#63802](https://github.com/ClickHouse/ClickHouse/pull/63802) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Support INSERT with VALUES in the ANTLR syntax file. [#63811](https://github.com/ClickHouse/ClickHouse/pull/63811) ([GG Bond](https://github.com/zzyReal666)).
+* Fix race in `ReplicatedMergeTreeLogEntryData`. [#63816](https://github.com/ClickHouse/ClickHouse/pull/63816) ([Antonio Andelic](https://github.com/antonio2368)).
+* Allow allocation during job destructor in `ThreadPool`. [#63829](https://github.com/ClickHouse/ClickHouse/pull/63829) ([Antonio Andelic](https://github.com/antonio2368)).
+* Remove HDFS from disks config for one integration test for arm. [#61457](https://github.com/ClickHouse/ClickHouse/issues/61457). [#63832](https://github.com/ClickHouse/ClickHouse/pull/63832) ([alesapin](https://github.com/alesapin)).
+* io_uring: add basic io_uring clickhouse perf test. [#63835](https://github.com/ClickHouse/ClickHouse/pull/63835) ([Tomer Shafir](https://github.com/tomershafir)).
+* Bump version for old image in test_short_strings_aggregation to make it work on arm. [#61457](https://github.com/ClickHouse/ClickHouse/issues/61457). [#63836](https://github.com/ClickHouse/ClickHouse/pull/63836) ([alesapin](https://github.com/alesapin)).
+* fix typo. [#63838](https://github.com/ClickHouse/ClickHouse/pull/63838) ([Alexander Gololobov](https://github.com/davenger)).
+* Disable test `test_non_default_compression/test.py::test_preconfigured_deflateqpl_codec` on arm. [#61457](https://github.com/ClickHouse/ClickHouse/issues/61457). [#63839](https://github.com/ClickHouse/ClickHouse/pull/63839) ([alesapin](https://github.com/alesapin)).
+* This PR was reverted. [#63857](https://github.com/ClickHouse/ClickHouse/pull/63857) ([Sema Checherinda](https://github.com/CheSema)).
+* Remove unnecessary logging statements in MergeJoinTransform.cpp. [#63860](https://github.com/ClickHouse/ClickHouse/pull/63860) ([vdimir](https://github.com/vdimir)).
+* Temporary disables 3 integration tcs on arm until https://github.com/clickhouse/clickhouse/issues/63855 is resolved. [#63867](https://github.com/ClickHouse/ClickHouse/pull/63867) ([Max K.](https://github.com/maxknv)).
+* Fix some settings values in 02455_one_row_from_csv_memory_usage test to make it less flaky. [#63874](https://github.com/ClickHouse/ClickHouse/pull/63874) ([Kruglov Pavel](https://github.com/Avogar)).
+* Randomise `allow_experimental_parallel_reading_from_replicas` in stress tests. [#63899](https://github.com/ClickHouse/ClickHouse/pull/63899) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix logs test for binary data by converting it to a valid UTF8 string. [#63909](https://github.com/ClickHouse/ClickHouse/pull/63909) ([Alexey Katsman](https://github.com/alexkats)).
+* More sanity checks for parallel replicas. [#63910](https://github.com/ClickHouse/ClickHouse/pull/63910) ([Nikita Taranov](https://github.com/nickitat)).
+* Include checks like `Stateless tests (asan, distributed cache, meta storage in keeper, s3 storage) [2/3]` in `Mergeable Check` and `A Sync`. [#63945](https://github.com/ClickHouse/ClickHouse/pull/63945) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Insignificant libunwind build fixes. [#63946](https://github.com/ClickHouse/ClickHouse/pull/63946) ([Azat Khuzhin](https://github.com/azat)).
+* Revert multiline pretty changes due to performance problems. [#63947](https://github.com/ClickHouse/ClickHouse/pull/63947) ([Raúl Marín](https://github.com/Algunenano)).
+* Some usability improvements for c++expr script. [#63948](https://github.com/ClickHouse/ClickHouse/pull/63948) ([Azat Khuzhin](https://github.com/azat)).
+* Fix 02124_insert_deduplication_token_multiple_blocks. [#63950](https://github.com/ClickHouse/ClickHouse/pull/63950) ([Han Fei](https://github.com/hanfei1991)).
+* CI: aarch64: disable arm integration tests with kerberaized kafka. [#63961](https://github.com/ClickHouse/ClickHouse/pull/63961) ([Max K.](https://github.com/maxknv)).
+* Make events like [timeouts](https://play.clickhouse.com/play?user=play#U0VMRUNUICogRlJPTSBjaGVja3MgV0hFUkUgdGVzdF9uYW1lID09ICdDaGVjayB0aW1lb3V0IGV4cGlyZWQnIEFORCBjaGVja19zdGFydF90aW1lIEJFVFdFRU4gdG9EYXRlKCcyMDI0LTA1LTEwJykgQU5EIHRvRGF0ZSgnMjAyNC0wNS0xNScp) visible in CI DB. [#63982](https://github.com/ClickHouse/ClickHouse/pull/63982) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Slightly better setting `force_optimize_projection_name`. [#63997](https://github.com/ClickHouse/ClickHouse/pull/63997) ([Anton Popov](https://github.com/CurtizJ)).
+* chore(ci-workers): remove reusable from tailscale key. [#63999](https://github.com/ClickHouse/ClickHouse/pull/63999) ([Gabriel Martinez](https://github.com/GMartinez-Sisti)).
+* Better script to collect symbols statistics. [#64013](https://github.com/ClickHouse/ClickHouse/pull/64013) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix a typo in Analyzer. [#64022](https://github.com/ClickHouse/ClickHouse/pull/64022) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix libbcrypt for FreeBSD build. [#64023](https://github.com/ClickHouse/ClickHouse/pull/64023) ([Azat Khuzhin](https://github.com/azat)).
+* Remove some unnecessary `UNREACHABLE`s. [#64035](https://github.com/ClickHouse/ClickHouse/pull/64035) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add `ClickHouseVersion.copy` method. Create a branch release in advance without spinning out the release to increase the stability. [#64039](https://github.com/ClickHouse/ClickHouse/pull/64039) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix searching for libclang_rt.builtins.*.a on FreeBSD. [#64051](https://github.com/ClickHouse/ClickHouse/pull/64051) ([Azat Khuzhin](https://github.com/azat)).
+* The mime type is not 100% reliable for Python and shell scripts without shebangs; add a check for file extension. [#64062](https://github.com/ClickHouse/ClickHouse/pull/64062) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix waiting for mutations with retriable errors. [#64063](https://github.com/ClickHouse/ClickHouse/pull/64063) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* harmonize h3PointDist* error messages. [#64080](https://github.com/ClickHouse/ClickHouse/pull/64080) ([Yohann Jardin](https://github.com/yohannj)).
+* This log message is better in Trace. [#64081](https://github.com/ClickHouse/ClickHouse/pull/64081) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Prevent stack overflow in Fuzzer and Stress test. [#64082](https://github.com/ClickHouse/ClickHouse/pull/64082) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* tests: fix expected error for 03036_reading_s3_archives (fixes CI). [#64089](https://github.com/ClickHouse/ClickHouse/pull/64089) ([Azat Khuzhin](https://github.com/azat)).
+* Fix sanitizers. [#64090](https://github.com/ClickHouse/ClickHouse/pull/64090) ([Azat Khuzhin](https://github.com/azat)).
+* Update llvm/clang to 18.1.6. [#64091](https://github.com/ClickHouse/ClickHouse/pull/64091) ([Azat Khuzhin](https://github.com/azat)).
+* Set green Mergeable Check status only after all required checks are passed with success - All non-required checks are started at stage Test_3 when all required checks are passed in Test_1/2. [#64093](https://github.com/ClickHouse/ClickHouse/pull/64093) ([Max K.](https://github.com/maxknv)).
+* Move `isAllASCII` from UTFHelper to StringUtils. [#64108](https://github.com/ClickHouse/ClickHouse/pull/64108) ([Robert Schulze](https://github.com/rschu1ze)).
+* Throw out some `inline`s. [#64110](https://github.com/ClickHouse/ClickHouse/pull/64110) ([Robert Schulze](https://github.com/rschu1ze)).
+* Clean up .clang-tidy after transition to Clang 18. [#64111](https://github.com/ClickHouse/ClickHouse/pull/64111) ([Robert Schulze](https://github.com/rschu1ze)).
+* Ignore exception when checking for cgroupsv2. [#64118](https://github.com/ClickHouse/ClickHouse/pull/64118) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add retries in git submodule update. [#64125](https://github.com/ClickHouse/ClickHouse/pull/64125) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* See https://s3.amazonaws.com/clickhouse-test-reports/63946/86cf1e13d866333b8a511badd7f2fe186d810646/ast_fuzzer__ubsan_.html. [#64127](https://github.com/ClickHouse/ClickHouse/pull/64127) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Refactoring of Server.h: Isolate server management from other logic. [#64132](https://github.com/ClickHouse/ClickHouse/pull/64132) ([TTPO100AJIEX](https://github.com/TTPO100AJIEX)).
+* Syncing code. [#64135](https://github.com/ClickHouse/ClickHouse/pull/64135) ([Antonio Andelic](https://github.com/antonio2368)).
+* Losen build resource limits for unusual architectures. [#64152](https://github.com/ClickHouse/ClickHouse/pull/64152) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* fix clang tidy. [#64179](https://github.com/ClickHouse/ClickHouse/pull/64179) ([Han Fei](https://github.com/hanfei1991)).
+* Fix: 02124_insert_deduplication_token_multiple_blocks_replica. [#64181](https://github.com/ClickHouse/ClickHouse/pull/64181) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix global query profiler. [#64187](https://github.com/ClickHouse/ClickHouse/pull/64187) ([Azat Khuzhin](https://github.com/azat)).
+* CI: cancel running PR wf after adding to MQ. [#64188](https://github.com/ClickHouse/ClickHouse/pull/64188) ([Max K.](https://github.com/maxknv)).
+* Add profile events for number of rows read during/after prewhere. [#64198](https://github.com/ClickHouse/ClickHouse/pull/64198) ([Nikita Taranov](https://github.com/nickitat)).
+* Add debug logging to EmbeddedRocksDBBulkSink. [#64203](https://github.com/ClickHouse/ClickHouse/pull/64203) ([vdimir](https://github.com/vdimir)).
+* Fix special builds (due to excessive resource usage - memory/CPU). [#64204](https://github.com/ClickHouse/ClickHouse/pull/64204) ([Azat Khuzhin](https://github.com/azat)).
+* Update InterpreterCreateQuery.cpp. [#64207](https://github.com/ClickHouse/ClickHouse/pull/64207) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Remove unused storage_snapshot field from MergeTreeSelectProcessor. [#64217](https://github.com/ClickHouse/ClickHouse/pull/64217) ([Alexander Gololobov](https://github.com/davenger)).
+* Add test for [#37090](https://github.com/ClickHouse/ClickHouse/issues/37090). [#64220](https://github.com/ClickHouse/ClickHouse/pull/64220) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Small cli tool. [#64227](https://github.com/ClickHouse/ClickHouse/pull/64227) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Make `settings_changes_history` const. [#64230](https://github.com/ClickHouse/ClickHouse/pull/64230) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* test for [#45804](https://github.com/ClickHouse/ClickHouse/issues/45804). [#64245](https://github.com/ClickHouse/ClickHouse/pull/64245) ([Denny Crane](https://github.com/den-crane)).
+* Update version after release. [#64283](https://github.com/ClickHouse/ClickHouse/pull/64283) ([Raúl Marín](https://github.com/Algunenano)).
+* Followup for [#63691](https://github.com/ClickHouse/ClickHouse/issues/63691). [#64285](https://github.com/ClickHouse/ClickHouse/pull/64285) ([vdimir](https://github.com/vdimir)).
+* CI: dependency fix for changelog.py. [#64293](https://github.com/ClickHouse/ClickHouse/pull/64293) ([Max K.](https://github.com/maxknv)).
+* Print query in explain plan with parallel replicas. [#64298](https://github.com/ClickHouse/ClickHouse/pull/64298) ([vdimir](https://github.com/vdimir)).
+* CI: Cancel sync wf on new push. [#64299](https://github.com/ClickHouse/ClickHouse/pull/64299) ([Max K.](https://github.com/maxknv)).
+* CI: master workflow with folded jobs. [#64340](https://github.com/ClickHouse/ClickHouse/pull/64340) ([Max K.](https://github.com/maxknv)).
+* CI: Sync, Merge check, CI gh's statuses fixes. [#64348](https://github.com/ClickHouse/ClickHouse/pull/64348) ([Max K.](https://github.com/maxknv)).
+* Enable 02494_query_cache_nested_query_bug for Analyzer. [#64357](https://github.com/ClickHouse/ClickHouse/pull/64357) ([Robert Schulze](https://github.com/rschu1ze)).
+* Rename allow_deprecated_functions to allow_deprecated_error_prone_window_functions. [#64358](https://github.com/ClickHouse/ClickHouse/pull/64358) ([Raúl Marín](https://github.com/Algunenano)).
+* Change input_format_parquet_use_native_reader to 24.6. [#64359](https://github.com/ClickHouse/ClickHouse/pull/64359) ([Raúl Marín](https://github.com/Algunenano)).
+* Update description for settings `cross_join_min_rows_to_compress` and `cross_join_min_bytes_to_compress`. [#64360](https://github.com/ClickHouse/ClickHouse/pull/64360) ([Nikita Fomichev](https://github.com/fm4v)).
+* Changed the unreleased setting `aggregate_function_group_array_has_limit_size` to `aggregate_function_group_array_action_when_limit_is_reached`. [#64362](https://github.com/ClickHouse/ClickHouse/pull/64362) ([Raúl Marín](https://github.com/Algunenano)).
+* Split tests 03039_dynamic_all_merge_algorithms to avoid timeouts. [#64363](https://github.com/ClickHouse/ClickHouse/pull/64363) ([Kruglov Pavel](https://github.com/Avogar)).
+* Try to fix GWPAsan. [#64365](https://github.com/ClickHouse/ClickHouse/pull/64365) ([Antonio Andelic](https://github.com/antonio2368)).
+* CI: add secrets to reusable stage wf yml. [#64366](https://github.com/ClickHouse/ClickHouse/pull/64366) ([Max K.](https://github.com/maxknv)).
+* Do not run tests tagged 'no-s3-storage-with-slow-build' with ASan. [#64367](https://github.com/ClickHouse/ClickHouse/pull/64367) ([vdimir](https://github.com/vdimir)).
+* This change was reverted. [#64386](https://github.com/ClickHouse/ClickHouse/pull/64386) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)).
+* Update s3queue.md. [#64389](https://github.com/ClickHouse/ClickHouse/pull/64389) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* test for [#64211](https://github.com/ClickHouse/ClickHouse/issues/64211). [#64390](https://github.com/ClickHouse/ClickHouse/pull/64390) ([Denny Crane](https://github.com/den-crane)).
+* Follow-up to [#59767](https://github.com/ClickHouse/ClickHouse/issues/59767). [#64398](https://github.com/ClickHouse/ClickHouse/pull/64398) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Remove wrong comment. [#64403](https://github.com/ClickHouse/ClickHouse/pull/64403) ([Sergei Trifonov](https://github.com/serxa)).
+* Follow up to [#59767](https://github.com/ClickHouse/ClickHouse/issues/59767). [#64404](https://github.com/ClickHouse/ClickHouse/pull/64404) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Refactor s3 settings (move settings parsing into single place). [#64412](https://github.com/ClickHouse/ClickHouse/pull/64412) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* This PR was reverted. [#64423](https://github.com/ClickHouse/ClickHouse/pull/64423) ([Sergei Trifonov](https://github.com/serxa)).
+* Fix test after [#64404](https://github.com/ClickHouse/ClickHouse/issues/64404). [#64432](https://github.com/ClickHouse/ClickHouse/pull/64432) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Faster TestKeeper shutdown. [#64433](https://github.com/ClickHouse/ClickHouse/pull/64433) ([Alexander Gololobov](https://github.com/davenger)).
+* Remove some logging. [#64434](https://github.com/ClickHouse/ClickHouse/pull/64434) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Revert "Revert "Remove some unnecessary UNREACHABLEs"". [#64435](https://github.com/ClickHouse/ClickHouse/pull/64435) ([Robert Schulze](https://github.com/rschu1ze)).
+* Clean settings in 02943_variant_read_subcolumns test. [#64437](https://github.com/ClickHouse/ClickHouse/pull/64437) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add a comment after [#64226](https://github.com/ClickHouse/ClickHouse/issues/64226). [#64449](https://github.com/ClickHouse/ClickHouse/pull/64449) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* CI: fix build_report selection in case of job reuse. [#64459](https://github.com/ClickHouse/ClickHouse/pull/64459) ([Max K.](https://github.com/maxknv)).
+* Add Critical bugfix category in PR template. [#64480](https://github.com/ClickHouse/ClickHouse/pull/64480) ([Max K.](https://github.com/maxknv)).
+* Remove `generateSnowflakeIDThreadMonotonic`. [#64499](https://github.com/ClickHouse/ClickHouse/pull/64499) ([Robert Schulze](https://github.com/rschu1ze)).
+* Move analyzer attempt 2. [#64500](https://github.com/ClickHouse/ClickHouse/pull/64500) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Sync some code back from internal to public repository. [#64502](https://github.com/ClickHouse/ClickHouse/pull/64502) ([Robert Schulze](https://github.com/rschu1ze)).
+* Remove `generateUUIDv7(NonMonotonic|ThreadMonotonic)` functions. [#64506](https://github.com/ClickHouse/ClickHouse/pull/64506) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix bash completion for settings. [#64521](https://github.com/ClickHouse/ClickHouse/pull/64521) ([Azat Khuzhin](https://github.com/azat)).
+* Use max_read_buffer_size for file descriptors as well in file(). [#64532](https://github.com/ClickHouse/ClickHouse/pull/64532) ([Azat Khuzhin](https://github.com/azat)).
+* Temporarily disable `enable_vertical_final` setting by default. This feature should not be used in older releases because it [might crash](https://github.com/ClickHouse/ClickHouse/issues/64543), but it's already fixed in 24.6 where this setting change has been reverted and `enable_vertical_final` is again enabled by default. [#64544](https://github.com/ClickHouse/ClickHouse/pull/64544) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Removed excessive calls to `flush logs` and disabled under sanitizers. [#64550](https://github.com/ClickHouse/ClickHouse/pull/64550) ([Nikita Taranov](https://github.com/nickitat)).
+* Sync code moved in private repo back back to public repo. [#64551](https://github.com/ClickHouse/ClickHouse/pull/64551) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add support for custom type to ASTLiteral, or else the type may be lost when parse the ast. E.g. set a ASTLiteral to DataTime32 with value 19870, then it will be parsed to Int16. [#64562](https://github.com/ClickHouse/ClickHouse/pull/64562) ([shuai.xu](https://github.com/shuai-xu)).
+* Add a temporary known host for git over ssh. [#64569](https://github.com/ClickHouse/ClickHouse/pull/64569) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Cache first analysis result in ReadFromMergeTree. [#64579](https://github.com/ClickHouse/ClickHouse/pull/64579) ([Igor Nikonov](https://github.com/devcrafter)).
+* Derive script parameters (labels) from the --repo/--from-repo - fix to not create backports for all release branches if backport for specific branch only. [#64603](https://github.com/ClickHouse/ClickHouse/pull/64603) ([Max K.](https://github.com/maxknv)).
+* CI fixes. [#64605](https://github.com/ClickHouse/ClickHouse/pull/64605) ([Max K.](https://github.com/maxknv)).
+* Double-checking [#59318](https://github.com/ClickHouse/ClickHouse/issues/59318) and docs for `Map`. [#64606](https://github.com/ClickHouse/ClickHouse/pull/64606) ([Robert Schulze](https://github.com/rschu1ze)).
+* Update CHANGELOG.md. [#64609](https://github.com/ClickHouse/ClickHouse/pull/64609) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Tests: Convert numeric to symbolic error codes. [#64635](https://github.com/ClickHouse/ClickHouse/pull/64635) ([Robert Schulze](https://github.com/rschu1ze)).
+* Move NamedCollectionsFactory into a separate file. [#64642](https://github.com/ClickHouse/ClickHouse/pull/64642) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Shuffle tests for parallel execution. [#64646](https://github.com/ClickHouse/ClickHouse/pull/64646) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* CI: Do not upload binaries for special builds in PRs. [#64653](https://github.com/ClickHouse/ClickHouse/pull/64653) ([Max K.](https://github.com/maxknv)).
+* Update changelog. [#64654](https://github.com/ClickHouse/ClickHouse/pull/64654) ([Robert Schulze](https://github.com/rschu1ze)).
+* Parallel replicas: simple cleanup. [#64655](https://github.com/ClickHouse/ClickHouse/pull/64655) ([Igor Nikonov](https://github.com/devcrafter)).
+* Be more graceful with existing tables with `inverted` indexes. [#64656](https://github.com/ClickHouse/ClickHouse/pull/64656) ([Robert Schulze](https://github.com/rschu1ze)).
+* CI: Build Report Check to verify only enabled builds. [#64669](https://github.com/ClickHouse/ClickHouse/pull/64669) ([Max K.](https://github.com/maxknv)).
+* Tests: Convert error numbers to symbolic error codes, pt. II. [#64670](https://github.com/ClickHouse/ClickHouse/pull/64670) ([Robert Schulze](https://github.com/rschu1ze)).
+* Split query analyzer. [#64672](https://github.com/ClickHouse/ClickHouse/pull/64672) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* By the end of CI, CI_Running status must be SUCCESS or FAILURE never PENDING. [#64693](https://github.com/ClickHouse/ClickHouse/pull/64693) ([Max K.](https://github.com/maxknv)).
+* The following list of merged PRs is not present in the release branch and was added to the changelog by mistake:. [#64704](https://github.com/ClickHouse/ClickHouse/pull/64704) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* CI: MergeQueue: add binary_release and unit tests. [#64705](https://github.com/ClickHouse/ClickHouse/pull/64705) ([Max K.](https://github.com/maxknv)).
+* Fix to get first good enough GH token instead of getting and comparing all of them. [#64709](https://github.com/ClickHouse/ClickHouse/pull/64709) ([Max K.](https://github.com/maxknv)).
+* Check for missing Upload ID in CreateMultipartUpload reply. [#64714](https://github.com/ClickHouse/ClickHouse/pull/64714) ([Michael Kolupaev](https://github.com/al13n321)).
+* Update version_date.tsv and changelogs after v24.5.1.1763-stable. [#64715](https://github.com/ClickHouse/ClickHouse/pull/64715) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix (unreleased) `loop()` table function crashing on empty table name. [#64716](https://github.com/ClickHouse/ClickHouse/pull/64716) ([Michael Kolupaev](https://github.com/al13n321)).
+* Update CHANGELOG.md. [#64730](https://github.com/ClickHouse/ClickHouse/pull/64730) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* CI: ci.py refactoring. [#64734](https://github.com/ClickHouse/ClickHouse/pull/64734) ([Max K.](https://github.com/maxknv)).
+* Return the explanation for session moved error. [#64747](https://github.com/ClickHouse/ClickHouse/pull/64747) ([Antonio Andelic](https://github.com/antonio2368)).
+* Adjust the version_helper and script to a new release scheme. [#64759](https://github.com/ClickHouse/ClickHouse/pull/64759) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Do not try to write columns.txt if it does not exist for write-once storages. [#64762](https://github.com/ClickHouse/ClickHouse/pull/64762) ([Azat Khuzhin](https://github.com/azat)).
+* Update 02482_load_parts_refcounts.sh. [#64765](https://github.com/ClickHouse/ClickHouse/pull/64765) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix crash with DISTINCT and window functions. [#64767](https://github.com/ClickHouse/ClickHouse/pull/64767) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix assert in IObjectStorageIteratorAsync. [#64770](https://github.com/ClickHouse/ClickHouse/pull/64770) ([Michael Kolupaev](https://github.com/al13n321)).
+* Make table functions always report engine 'StorageProxy' in system.tables. [#64771](https://github.com/ClickHouse/ClickHouse/pull/64771) ([Michael Kolupaev](https://github.com/al13n321)).
+* Ask about company name on GitHub. [#64774](https://github.com/ClickHouse/ClickHouse/pull/64774) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix flaky tests about SQLite. [#64776](https://github.com/ClickHouse/ClickHouse/pull/64776) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Remove iostream debug helpers. [#64777](https://github.com/ClickHouse/ClickHouse/pull/64777) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Remove unnecessary comment. [#64785](https://github.com/ClickHouse/ClickHouse/pull/64785) ([Raúl Marín](https://github.com/Algunenano)).
+* Follow-ups to some PRs. [#64787](https://github.com/ClickHouse/ClickHouse/pull/64787) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Attempt to fix 02228_merge_tree_insert_memory_usage.sql flakiness for s3. [#64800](https://github.com/ClickHouse/ClickHouse/pull/64800) ([Raúl Marín](https://github.com/Algunenano)).
+* Add regression test for filter propagation through `Merge` engine. [#64806](https://github.com/ClickHouse/ClickHouse/pull/64806) ([Nikita Taranov](https://github.com/nickitat)).
+* Migrate changelog.py to a descendant of fuzzywuzzy. [#64807](https://github.com/ClickHouse/ClickHouse/pull/64807) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* A follow-up for https://github.com/ClickHouse/ClickHouse/pull/64039 and [#64759](https://github.com/ClickHouse/ClickHouse/issues/64759). [#64813](https://github.com/ClickHouse/ClickHouse/pull/64813) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Make row order optimization non-experimental. [#64814](https://github.com/ClickHouse/ClickHouse/pull/64814) ([Robert Schulze](https://github.com/rschu1ze)).
+* Didn't catch it at the time when all versions belonged to the current year. [#64817](https://github.com/ClickHouse/ClickHouse/pull/64817) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix clang-tidy build. [#64823](https://github.com/ClickHouse/ClickHouse/pull/64823) ([Robert Schulze](https://github.com/rschu1ze)).
+* Sets all builds that we run tests on to normal build list. [#64824](https://github.com/ClickHouse/ClickHouse/pull/64824) ([Max K.](https://github.com/maxknv)).
+* CI: fix CI await feature. [#64825](https://github.com/ClickHouse/ClickHouse/pull/64825) ([Max K.](https://github.com/maxknv)).
+* Fix clang-tidy. [#64827](https://github.com/ClickHouse/ClickHouse/pull/64827) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Upload blob_storage_log from stateless tests. [#64843](https://github.com/ClickHouse/ClickHouse/pull/64843) ([alesapin](https://github.com/alesapin)).
+* Follow-up to [#64349](https://github.com/ClickHouse/ClickHouse/issues/64349). [#64845](https://github.com/ClickHouse/ClickHouse/pull/64845) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Simplify handling of old 'inverted' indexes. [#64846](https://github.com/ClickHouse/ClickHouse/pull/64846) ([Robert Schulze](https://github.com/rschu1ze)).
+* Use issue templates defined in YAML provide more user-friendly experience. [#64850](https://github.com/ClickHouse/ClickHouse/pull/64850) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Handle logs from rocksdb by ClickHouse internal logging. [#64856](https://github.com/ClickHouse/ClickHouse/pull/64856) ([Azat Khuzhin](https://github.com/azat)).
+* Follow-up for https://github.com/ClickHouse/ClickHouse/pull/59357. [#64860](https://github.com/ClickHouse/ClickHouse/pull/64860) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* added mlock and mlockall to aspell-dict to be ignored. [#64863](https://github.com/ClickHouse/ClickHouse/pull/64863) ([Ali](https://github.com/xogoodnow)).
+* A tiny fix for fancy quotes. [#64883](https://github.com/ClickHouse/ClickHouse/pull/64883) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix possible loss of "Query was cancelled" message in client. [#64888](https://github.com/ClickHouse/ClickHouse/pull/64888) ([Azat Khuzhin](https://github.com/azat)).
+* We accidentally lost the way to set `PR Check` failure at some point. [#64890](https://github.com/ClickHouse/ClickHouse/pull/64890) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix global trace collector. [#64896](https://github.com/ClickHouse/ClickHouse/pull/64896) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix test_mask_sensitive_info/test.py::test_create_table. [#64901](https://github.com/ClickHouse/ClickHouse/pull/64901) ([Azat Khuzhin](https://github.com/azat)).
+* Update 03165_string_functions_with_token_text_indexes.sql. [#64903](https://github.com/ClickHouse/ClickHouse/pull/64903) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* When the branch is removed, it's impossible to get the diff by the labels. `print` in imported files spoils the `ipython` output. [#64904](https://github.com/ClickHouse/ClickHouse/pull/64904) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Disable transactions for unsupported storages even for materialized v…. [#64918](https://github.com/ClickHouse/ClickHouse/pull/64918) ([alesapin](https://github.com/alesapin)).
+* additional log for cleanupDetachedTables. [#64919](https://github.com/ClickHouse/ClickHouse/pull/64919) ([Konstantin Morozov](https://github.com/k-morozov)).
+* Fix tupleConcat of two empty tuples. This fixes [#64885](https://github.com/ClickHouse/ClickHouse/issues/64885). [#64923](https://github.com/ClickHouse/ClickHouse/pull/64923) ([Amos Bird](https://github.com/amosbird)).
+* CI: Minor fixes in ci scripts. [#64950](https://github.com/ClickHouse/ClickHouse/pull/64950) ([Max K.](https://github.com/maxknv)).
+* Fix error message (it was strange). [#64952](https://github.com/ClickHouse/ClickHouse/pull/64952) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Update fmtlib version to 9.1.0. [#64959](https://github.com/ClickHouse/ClickHouse/pull/64959) ([Duc Canh Le](https://github.com/canhld94)).
+* Test 02908_many_requests_to_system_replicas makes a lot of heavy requests and it overloads server if it's an ASAN build. [#64966](https://github.com/ClickHouse/ClickHouse/pull/64966) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix (unreleased) bug in short circuit evaluation. [#64967](https://github.com/ClickHouse/ClickHouse/pull/64967) ([Raúl Marín](https://github.com/Algunenano)).
+* Update version_date.tsv and changelogs after v24.4.2.141-stable. [#64968](https://github.com/ClickHouse/ClickHouse/pull/64968) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix `test_attach_partition_using_copy`. [#64977](https://github.com/ClickHouse/ClickHouse/pull/64977) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Faster processing of scheduler queue activations. [#64985](https://github.com/ClickHouse/ClickHouse/pull/64985) ([Sergei Trifonov](https://github.com/serxa)).
+* CI: Fix nightly workflow. [#64987](https://github.com/ClickHouse/ClickHouse/pull/64987) ([Max K.](https://github.com/maxknv)).
+* Fix innocuous data race in detectLanguage. [#64988](https://github.com/ClickHouse/ClickHouse/pull/64988) ([Raúl Marín](https://github.com/Algunenano)).
+* CI: Builds in CI settings. [#64994](https://github.com/ClickHouse/ClickHouse/pull/64994) ([Max K.](https://github.com/maxknv)).
+* REVERTED. [#65009](https://github.com/ClickHouse/ClickHouse/pull/65009) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* CI: Fix backports. [#65010](https://github.com/ClickHouse/ClickHouse/pull/65010) ([Max K.](https://github.com/maxknv)).
+* Try fix 03143_prewhere_profile_events. [#65014](https://github.com/ClickHouse/ClickHouse/pull/65014) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix 03165_string_functions_with_token_text_indexes. [#65018](https://github.com/ClickHouse/ClickHouse/pull/65018) ([Julia Kartseva](https://github.com/jkartseva)).
+* This change was reverted. [#65028](https://github.com/ClickHouse/ClickHouse/pull/65028) ([Sergei Trifonov](https://github.com/serxa)).
+* Bump googletest to latest HEAD. [#65038](https://github.com/ClickHouse/ClickHouse/pull/65038) ([Robert Schulze](https://github.com/rschu1ze)).
+* Improve comment about AsynchronousMetrics. [#65040](https://github.com/ClickHouse/ClickHouse/pull/65040) ([Antonio Andelic](https://github.com/antonio2368)).
+* CI: Remove fuzzer build from normal CI run (bugfix). [#65041](https://github.com/ClickHouse/ClickHouse/pull/65041) ([Max K.](https://github.com/maxknv)).
+* CI config refactoring. [#65045](https://github.com/ClickHouse/ClickHouse/pull/65045) ([Max K.](https://github.com/maxknv)).
+* Bump abseil to latest HEAD. [#65048](https://github.com/ClickHouse/ClickHouse/pull/65048) ([Robert Schulze](https://github.com/rschu1ze)).
+* Capture weak_ptr of ContextAccess for safety. [#65051](https://github.com/ClickHouse/ClickHouse/pull/65051) ([Alexander Gololobov](https://github.com/davenger)).
+* Stateless tests: add test for SIZES_OF_NESTED_COLUMNS_ARE_INCONSISTENT. [#65056](https://github.com/ClickHouse/ClickHouse/pull/65056) ([Nikita Fomichev](https://github.com/fm4v)).
+* Increase timeout in wait_for_all_mutations. [#65058](https://github.com/ClickHouse/ClickHouse/pull/65058) ([Alexander Gololobov](https://github.com/davenger)).
+* Tests for _time virtual column in file alike storages. [#65064](https://github.com/ClickHouse/ClickHouse/pull/65064) ([Ilya Golshtein](https://github.com/ilejn)).
+* Update odbc-bridge.md. [#65099](https://github.com/ClickHouse/ClickHouse/pull/65099) ([Alexander Gololobov](https://github.com/davenger)).
+* Small fix for 02340_parts_refcnt_mergetree. [#65105](https://github.com/ClickHouse/ClickHouse/pull/65105) ([Nikita Taranov](https://github.com/nickitat)).
+* Re-enable OpenSSL session caching. [#65111](https://github.com/ClickHouse/ClickHouse/pull/65111) ([Robert Schulze](https://github.com/rschu1ze)).
+* Update test_replicated_database/test.py. [#65112](https://github.com/ClickHouse/ClickHouse/pull/65112) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix false positives leaky memory warnings in OpenSSL. [#65125](https://github.com/ClickHouse/ClickHouse/pull/65125) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix `Initiator received more initial requests than there are replicas` with `loop` engine. [#65133](https://github.com/ClickHouse/ClickHouse/pull/65133) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix 'Tasks in BackgroundSchedulePool cannot throw' caused by MergeTreeData::loadUnexpectedDataParts(). [#65135](https://github.com/ClickHouse/ClickHouse/pull/65135) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix bad error message. [#65137](https://github.com/ClickHouse/ClickHouse/pull/65137) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Just fixing flaky unit tests. [#65152](https://github.com/ClickHouse/ClickHouse/pull/65152) ([Sema Checherinda](https://github.com/CheSema)).
+* This change was reverted. [#65164](https://github.com/ClickHouse/ClickHouse/pull/65164) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Ensure submodules are named consistently. [#65167](https://github.com/ClickHouse/ClickHouse/pull/65167) ([Robert Schulze](https://github.com/rschu1ze)).
+* Remove obsolete fix from aws submodule. [#65168](https://github.com/ClickHouse/ClickHouse/pull/65168) ([Robert Schulze](https://github.com/rschu1ze)).
+* CI: Fix not-merged cherry-picks for backports. [#65181](https://github.com/ClickHouse/ClickHouse/pull/65181) ([Max K.](https://github.com/maxknv)).
+* Add an assertion in ReplicatedMergeTreeQueue. [#65184](https://github.com/ClickHouse/ClickHouse/pull/65184) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix bug in unreleased code. [#65185](https://github.com/ClickHouse/ClickHouse/pull/65185) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix docs for skipping-indexes.md. [#65194](https://github.com/ClickHouse/ClickHouse/pull/65194) ([morning-color](https://github.com/morning-color)).
+* Fix the descriptions of some server settings. [#65200](https://github.com/ClickHouse/ClickHouse/pull/65200) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix issue after [#64813](https://github.com/ClickHouse/ClickHouse/issues/64813) with broken search in the changelog, and missing zstd in a style-check image. [#65202](https://github.com/ClickHouse/ClickHouse/pull/65202) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix bug in unreleased code. [#65203](https://github.com/ClickHouse/ClickHouse/pull/65203) ([Raúl Marín](https://github.com/Algunenano)).
+* Add test prewhere merge. [#65207](https://github.com/ClickHouse/ClickHouse/pull/65207) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Sync ProfileEvents.h. [#65208](https://github.com/ClickHouse/ClickHouse/pull/65208) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* FinishCheck to set failure if workflow failed. [#65228](https://github.com/ClickHouse/ClickHouse/pull/65228) ([Max K.](https://github.com/maxknv)).
+* Update version_date.tsv and changelogs after v24.3.4.147-lts. [#65235](https://github.com/ClickHouse/ClickHouse/pull/65235) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelogs after v24.5.3.5-stable. [#65240](https://github.com/ClickHouse/ClickHouse/pull/65240) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fails sometimes for debug build https://s3.amazonaws.com/clickhouse-test-reports/0/af6afd904316bfb771737faa147ce8aea72dd705/stateless_tests__debug__[4_5].html. [#65245](https://github.com/ClickHouse/ClickHouse/pull/65245) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix libunwind in CI. [#65247](https://github.com/ClickHouse/ClickHouse/pull/65247) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* CI: Do not skip FinishCheck in Merge Queue. [#65249](https://github.com/ClickHouse/ClickHouse/pull/65249) ([Max K.](https://github.com/maxknv)).
+* Add a test just in case. [#65271](https://github.com/ClickHouse/ClickHouse/pull/65271) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Disable 02581_share_big_sets_between_multiple_mutations_tasks_long in coverage run. [#65295](https://github.com/ClickHouse/ClickHouse/pull/65295) ([Alexander Gololobov](https://github.com/davenger)).
+* Update version_date.tsv and changelogs after v23.8.15.35-lts. [#65300](https://github.com/ClickHouse/ClickHouse/pull/65300) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* mute test test_query_is_canceled_with_inf_retries. [#65301](https://github.com/ClickHouse/ClickHouse/pull/65301) ([Sema Checherinda](https://github.com/CheSema)).
+* Fix silly typo that caused wrong tags messages. [#65307](https://github.com/ClickHouse/ClickHouse/pull/65307) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Save server data for failed stateless tests. [#65309](https://github.com/ClickHouse/ClickHouse/pull/65309) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix 01246_buffer_flush flakiness (by tuning timeouts). [#65310](https://github.com/ClickHouse/ClickHouse/pull/65310) ([Azat Khuzhin](https://github.com/azat)).
+* Remove outdated override in stress tests. [#65323](https://github.com/ClickHouse/ClickHouse/pull/65323) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix bad code in `system.session_log`. [#65332](https://github.com/ClickHouse/ClickHouse/pull/65332) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* add tests for 'boom filter index with map'. [#65333](https://github.com/ClickHouse/ClickHouse/pull/65333) ([iceFireser](https://github.com/iceFireser)).
+* Fix crash in 03036_dynamic_read_subcolumns. [#65341](https://github.com/ClickHouse/ClickHouse/pull/65341) ([Kruglov Pavel](https://github.com/Avogar)).
+* Move tests 02942_variant_cast and 02944_variant_as_common_type to analyzer_tech_debt.txt. [#65342](https://github.com/ClickHouse/ClickHouse/pull/65342) ([Kruglov Pavel](https://github.com/Avogar)).
+* REVERTED. [#65384](https://github.com/ClickHouse/ClickHouse/pull/65384) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* CI: Add Non-blocking (Woolen wolfdog) CI mode. [#65385](https://github.com/ClickHouse/ClickHouse/pull/65385) ([Max K.](https://github.com/maxknv)).
+* Fix compatibility release check. [#65394](https://github.com/ClickHouse/ClickHouse/pull/65394) ([Alexey Katsman](https://github.com/alexkats)).
+* Move a leaksan suppression from Poco into OpenSSL. [#65396](https://github.com/ClickHouse/ClickHouse/pull/65396) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix tidy build. [#65415](https://github.com/ClickHouse/ClickHouse/pull/65415) ([Sergei Trifonov](https://github.com/serxa)).
+* Remove Tests dependency on Builds_2. No tests depend on Builds_2. [#65416](https://github.com/ClickHouse/ClickHouse/pull/65416) ([Max K.](https://github.com/maxknv)).
+* CI: PR workflow dependencies fix. [#65442](https://github.com/ClickHouse/ClickHouse/pull/65442) ([Max K.](https://github.com/maxknv)).
+* Fix test_storage_s3_queue/test.py::test_max_set_age. [#65452](https://github.com/ClickHouse/ClickHouse/pull/65452) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* CI: Rename A Sync status. [#65456](https://github.com/ClickHouse/ClickHouse/pull/65456) ([Max K.](https://github.com/maxknv)).
+* CI: Rename sync status. [#65464](https://github.com/ClickHouse/ClickHouse/pull/65464) ([Max K.](https://github.com/maxknv)).
+* This change was reverted. [#65466](https://github.com/ClickHouse/ClickHouse/pull/65466) ([Sergei Trifonov](https://github.com/serxa)).
+* Remove a feature wasn't part of any release yet. [#65480](https://github.com/ClickHouse/ClickHouse/pull/65480) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#65657](https://github.com/ClickHouse/ClickHouse/issues/65657): Fix of `PlanSquashingTransform`: pipeline stuck. [#65487](https://github.com/ClickHouse/ClickHouse/pull/65487) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Backported in [#65504](https://github.com/ClickHouse/ClickHouse/issues/65504): Fix bad test `02922_deduplication_with_zero_copy`. [#65492](https://github.com/ClickHouse/ClickHouse/pull/65492) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#65591](https://github.com/ClickHouse/ClickHouse/issues/65591): Setting `uniform_snowflake_conversion_functions` (not in any release yet) was replaced by setting `allow_deprecated_snowflake_conversion_functions`. The latter controls if the legacy snowflake conversion functions are available (by default, they are not). [#65522](https://github.com/ClickHouse/ClickHouse/pull/65522) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#65759](https://github.com/ClickHouse/ClickHouse/issues/65759): Renames Build report jobs. [#65554](https://github.com/ClickHouse/ClickHouse/pull/65554) ([Max K.](https://github.com/maxknv)).
+* Backported in [#65773](https://github.com/ClickHouse/ClickHouse/issues/65773): `base64En/Decode64Url` --> `base64En/Decode64URL`. [#65760](https://github.com/ClickHouse/ClickHouse/pull/65760) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#65805](https://github.com/ClickHouse/ClickHouse/issues/65805): CI: Fix for Builds report job in backports and releases. [#65774](https://github.com/ClickHouse/ClickHouse/pull/65774) ([Max K.](https://github.com/maxknv)).
+
diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md
index c348eb5ca07..c283cfbf4c2 100644
--- a/docs/en/development/continuous-integration.md
+++ b/docs/en/development/continuous-integration.md
@@ -91,6 +91,9 @@ cd ./utils/check-style
# Check python type hinting with mypy
./check-mypy
+# Check python with flake8
+./check-flake8
+
# Check code with codespell
./check-typos
diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md
index ec5760541e8..0a1fe58b16f 100644
--- a/docs/en/development/developer-instruction.md
+++ b/docs/en/development/developer-instruction.md
@@ -267,7 +267,7 @@ A pull request can be created even if the work is not completed yet. In this cas
Testing will commence as soon as ClickHouse employees label your PR with a tag “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour.
-The system will prepare ClickHouse binary builds for your pull request individually. To retrieve these builds click the “Details” link next to “ClickHouse build check” entry in the list of checks. There you will find direct links to the built .deb packages of ClickHouse which you can deploy even on your production servers (if you have no fear).
+The system will prepare ClickHouse binary builds for your pull request individually. To retrieve these builds click the “Details” link next to “Builds” entry in the list of checks. There you will find direct links to the built .deb packages of ClickHouse which you can deploy even on your production servers (if you have no fear).
Most probably some of the builds will fail at first times. This is due to the fact that we check builds both with gcc as well as with clang, with almost all of existing warnings (always with the `-Werror` flag) enabled for clang. On that same page, you can find all of the build logs so that you do not have to build ClickHouse in all of the possible ways.
diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md
index bbc7dac0a2a..269995a1a96 100644
--- a/docs/en/development/tests.md
+++ b/docs/en/development/tests.md
@@ -28,7 +28,7 @@ run, for example, the test `01428_hash_set_nan_key`, change to the repository
folder and run the following command:
```
-PATH=$PATH: tests/clickhouse-test 01428_hash_set_nan_key
+PATH=:$PATH tests/clickhouse-test 01428_hash_set_nan_key
```
Test results (`stderr` and `stdout`) are written to files `01428_hash_set_nan_key.[stderr|stdout]` which
@@ -229,6 +229,10 @@ For production builds, clang is used, but we also test make gcc builds. For deve
## Sanitizers {#sanitizers}
+:::note
+If the process (ClickHouse server or client) crashes at startup when running it locally, you might need to disable address space layout randomization: `sudo sysctl kernel.randomize_va_space=0`
+:::
+
### Address sanitizer
We run functional, integration, stress and unit tests under ASan on per-commit basis.
diff --git a/docs/en/engines/table-engines/integrations/azureBlobStorage.md b/docs/en/engines/table-engines/integrations/azureBlobStorage.md
index 0843ff1ac47..bdf96832e9d 100644
--- a/docs/en/engines/table-engines/integrations/azureBlobStorage.md
+++ b/docs/en/engines/table-engines/integrations/azureBlobStorage.md
@@ -54,6 +54,16 @@ SELECT * FROM test_table;
- `_path` — Path to the file. Type: `LowCardinalty(String)`.
- `_file` — Name of the file. Type: `LowCardinalty(String)`.
- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the size is unknown, the value is `NULL`.
+- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
+
+## Authentication
+
+Currently there are 3 ways to authenticate:
+- `Managed Identity` - Can be used by providing an `endpoint`, `connection_string` or `storage_account_url`.
+- `SAS Token` - Can be used by providing an `endpoint`, `connection_string` or `storage_account_url`. It is identified by presence of '?' in the url.
+- `Workload Identity` - Can be used by providing an `endpoint` or `storage_account_url`. If `use_workload_identity` parameter is set in config, ([workload identity](https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity#authenticate-azure-hosted-applications)) is used for authentication.
+
+
## See also
diff --git a/docs/en/engines/table-engines/integrations/hdfs.md b/docs/en/engines/table-engines/integrations/hdfs.md
index 2749fa7e479..c9df713231a 100644
--- a/docs/en/engines/table-engines/integrations/hdfs.md
+++ b/docs/en/engines/table-engines/integrations/hdfs.md
@@ -235,6 +235,7 @@ libhdfs3 support HDFS namenode HA.
- `_path` — Path to the file. Type: `LowCardinalty(String)`.
- `_file` — Name of the file. Type: `LowCardinalty(String)`.
- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the size is unknown, the value is `NULL`.
+- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
## Storage Settings {#storage-settings}
diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md
index 9d6395f73ac..21fdbc0b1a5 100644
--- a/docs/en/engines/table-engines/integrations/iceberg.md
+++ b/docs/en/engines/table-engines/integrations/iceberg.md
@@ -37,7 +37,7 @@ Using named collections:
http://test.s3.amazonaws.com/clickhouse-bucket/
- test
+ test
test
diff --git a/docs/en/engines/table-engines/integrations/mongodb.md b/docs/en/engines/table-engines/integrations/mongodb.md
index f87e8da8b5b..5bb3bc752f5 100644
--- a/docs/en/engines/table-engines/integrations/mongodb.md
+++ b/docs/en/engines/table-engines/integrations/mongodb.md
@@ -34,10 +34,11 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name
- `options` — MongoDB connection string options (optional parameter).
:::tip
-If you are using the MongoDB Atlas cloud offering please add these options:
+If you are using the MongoDB Atlas cloud offering:
```
-'connectTimeoutMS=10000&ssl=true&authSource=admin'
+- connection url can be obtained from 'Atlas SQL' option
+- use options: 'connectTimeoutMS=10000&ssl=true&authSource=admin'
```
:::
diff --git a/docs/en/engines/table-engines/integrations/nats.md b/docs/en/engines/table-engines/integrations/nats.md
index 9f7409a6893..78ce537224c 100644
--- a/docs/en/engines/table-engines/integrations/nats.md
+++ b/docs/en/engines/table-engines/integrations/nats.md
@@ -58,7 +58,7 @@ Optional parameters:
- `nats_max_reconnect` – Maximum amount of reconnection attempts per try to connect to NATS. Default: `5`.
- `nats_reconnect_wait` – Amount of time in milliseconds to sleep between each reconnect attempt. Default: `5000`.
- `nats_server_list` - Server list for connection. Can be specified to connect to NATS cluster.
-- `nats_skip_broken_messages` - NATS message parser tolerance to schema-incompatible messages per block. Default: `0`. If `nats_skip_broken_messages = N` then the engine skips *N* RabbitMQ messages that cannot be parsed (a message equals a row of data).
+- `nats_skip_broken_messages` - NATS message parser tolerance to schema-incompatible messages per block. Default: `0`. If `nats_skip_broken_messages = N` then the engine skips *N* NATS messages that cannot be parsed (a message equals a row of data).
- `nats_max_block_size` - Number of row collected by poll(s) for flushing data from NATS. Default: [max_insert_block_size](../../../operations/settings/settings.md#max_insert_block_size).
- `nats_flush_interval_ms` - Timeout for flushing data read from NATS. Default: [stream_flush_interval_ms](../../../operations/settings/settings.md#stream-flush-interval-ms).
- `nats_username` - NATS username.
@@ -67,7 +67,7 @@ Optional parameters:
- `nats_credential_file` - Path to a NATS credentials file.
- `nats_startup_connect_tries` - Number of connect tries at startup. Default: `5`.
- `nats_max_rows_per_message` — The maximum number of rows written in one NATS message for row-based formats. (default : `1`).
-- `nats_handle_error_mode` — How to handle errors for RabbitMQ engine. Possible values: default (the exception will be thrown if we fail to parse a message), stream (the exception message and raw message will be saved in virtual columns `_error` and `_raw_message`).
+- `nats_handle_error_mode` — How to handle errors for NATS engine. Possible values: default (the exception will be thrown if we fail to parse a message), stream (the exception message and raw message will be saved in virtual columns `_error` and `_raw_message`).
SSL connection:
diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md
index cb1da1c8e68..93f4a187656 100644
--- a/docs/en/engines/table-engines/integrations/s3.md
+++ b/docs/en/engines/table-engines/integrations/s3.md
@@ -53,14 +53,14 @@ For partitioning by month, use the `toYYYYMM(date_column)` expression, where `da
This example uses the [docker compose recipe](https://github.com/ClickHouse/examples/tree/5fdc6ff72f4e5137e23ea075c88d3f44b0202490/docker-compose-recipes/recipes/ch-and-minio-S3), which integrates ClickHouse and MinIO. You should be able to reproduce the same queries using S3 by replacing the endpoint and authentication values.
-Notice that the S3 endpoint in the `ENGINE` configuration uses the parameter token `{_partition_id}` as part of the S3 object (filename), and that the SELECT queries select against those resulting object names (e.g., `test_3.csv`).
+Notice that the S3 endpoint in the `ENGINE` configuration uses the parameter token `{_partition_id}` as part of the S3 object (filename), and that the SELECT queries select against those resulting object names (e.g., `test_3.csv`).
:::note
As shown in the example, querying from S3 tables that are partitioned is
not directly supported at this time, but can be accomplished by querying the individual partitions
using the S3 table function.
-The primary use-case for writing
+The primary use-case for writing
partitioned data in S3 is to enable transferring that data into another
ClickHouse system (for example, moving from on-prem systems to ClickHouse
Cloud). Because ClickHouse datasets are often very large, and network
@@ -78,9 +78,9 @@ CREATE TABLE p
)
ENGINE = S3(
# highlight-next-line
- 'http://minio:10000/clickhouse//test_{_partition_id}.csv',
- 'minioadmin',
- 'minioadminpassword',
+ 'http://minio:10000/clickhouse//test_{_partition_id}.csv',
+ 'minioadmin',
+ 'minioadminpassword',
'CSV')
PARTITION BY column3
```
@@ -145,6 +145,7 @@ Code: 48. DB::Exception: Received from localhost:9000. DB::Exception: Reading fr
- `_path` — Path to the file. Type: `LowCardinalty(String)`.
- `_file` — Name of the file. Type: `LowCardinalty(String)`.
- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the size is unknown, the value is `NULL`.
+- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
For more information about virtual columns see [here](../../../engines/table-engines/index.md#table_engines-virtual_columns).
diff --git a/docs/en/engines/table-engines/integrations/s3queue.md b/docs/en/engines/table-engines/integrations/s3queue.md
index aa7fa512480..11181703645 100644
--- a/docs/en/engines/table-engines/integrations/s3queue.md
+++ b/docs/en/engines/table-engines/integrations/s3queue.md
@@ -13,7 +13,7 @@ This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ec
CREATE TABLE s3_queue_engine_table (name String, value UInt32)
ENGINE = S3Queue(path, [NOSIGN, | aws_access_key_id, aws_secret_access_key,] format, [compression])
[SETTINGS]
- [mode = 'unordered',]
+ [mode = '',]
[after_processing = 'keep',]
[keeper_path = '',]
[s3queue_loading_retries = 0,]
@@ -28,6 +28,8 @@ CREATE TABLE s3_queue_engine_table (name String, value UInt32)
[s3queue_cleanup_interval_max_ms = 30000,]
```
+Starting with `24.7` settings without `s3queue_` prefix are also supported.
+
**Engine parameters**
- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `**`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [below](#wildcards-in-path).
@@ -75,7 +77,7 @@ Possible values:
- unordered — With unordered mode, the set of all already processed files is tracked with persistent nodes in ZooKeeper.
- ordered — With ordered mode, only the max name of the successfully consumed file, and the names of files that will be retried after unsuccessful loading attempt are being stored in ZooKeeper.
-Default value: `unordered`.
+Default value: `ordered` in versions before 24.6. Starting with 24.6 there is no default value, the setting becomes required to be specified manually. For tables created on earlier versions the default value will remain `Ordered` for compatibility.
### after_processing {#after_processing}
@@ -181,6 +183,10 @@ For 'Ordered' mode. Defines a maximum boundary for reschedule interval for a bac
Default value: `30000`.
+### s3queue_buckets {#buckets}
+
+For 'Ordered' mode. Available since `24.6`. If there are several replicas of S3Queue table, each working with the same metadata directory in keeper, the value of `s3queue_buckets` needs to be equal to at least the number of replicas. If `s3queue_processing_threads` setting is used as well, it makes sense to increase the value of `s3queue_buckets` setting even further, as it defines the actual parallelism of `S3Queue` processing.
+
## S3-related Settings {#s3-settings}
Engine supports all s3 related settings. For more information about S3 settings see [here](../../../engines/table-engines/integrations/s3.md).
@@ -267,7 +273,7 @@ For introspection use `system.s3queue` stateless table and `system.s3queue_log`
`exception` String
)
ENGINE = SystemS3Queue
-COMMENT 'SYSTEM TABLE is built on the fly.' │
+COMMENT 'Contains in-memory state of S3Queue metadata and currently processed rows per file.' │
└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md
index 689c05a24af..f0c4e1b0e34 100644
--- a/docs/en/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md
@@ -6,41 +6,32 @@ sidebar_label: MergeTree
# MergeTree
-The `MergeTree` engine and other engines of this family (`*MergeTree`) are the most commonly used and most robust ClickHouse table engines.
+The `MergeTree` engine and other engines of the `MergeTree` family (e.g. `ReplacingMergeTree`, `AggregatingMergeTree` ) are the most commonly used and most robust table engines in ClickHouse.
-Engines in the `MergeTree` family are designed for inserting a very large amount of data into a table. The data is quickly written to the table part by part, then rules are applied for merging the parts in the background. This method is much more efficient than continually rewriting the data in storage during insert.
+`MergeTree`-family table engines are designed for high data ingest rates and huge data volumes.
+Insert operations create table parts which are merged by a background process with other table parts.
-Main features:
+Main features of `MergeTree`-family table engines.
-- Stores data sorted by primary key.
+- The table's primary key determines the sort order within each table part (clustered index). The primary key also does not reference individual rows but blocks of 8192 rows called granules. This makes primary keys of huge data sets small enough to remain loaded in main memory, while still providing fast access to on-disk data.
- This allows you to create a small sparse index that helps find data faster.
+- Tables can be partitioned using an arbitrary partition expression. Partition pruning ensures partitions are omitted from reading when the query allows it.
-- Partitions can be used if the [partitioning key](/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md) is specified.
+- Data can be replicated across multiple cluster nodes for high availability, failover, and zero downtime upgrades. See [Data replication](/docs/en/engines/table-engines/mergetree-family/replication.md).
- ClickHouse supports certain operations with partitions that are more efficient than general operations on the same data with the same result. ClickHouse also automatically cuts off the partition data where the partitioning key is specified in the query.
+- `MergeTree` table engines support various statistics kinds and sampling methods to help query optimization.
-- Data replication support.
-
- The family of `ReplicatedMergeTree` tables provides data replication. For more information, see [Data replication](/docs/en/engines/table-engines/mergetree-family/replication.md).
-
-- Data sampling support.
-
- If necessary, you can set the data sampling method in the table.
-
-:::info
-The [Merge](/docs/en/engines/table-engines/special/merge.md/#merge) engine does not belong to the `*MergeTree` family.
+:::note
+Despite a similar name, the [Merge](/docs/en/engines/table-engines/special/merge.md/#merge) engine is different from `*MergeTree` engines.
:::
-If you need to update rows frequently, we recommend using the [`ReplacingMergeTree`](/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md) table engine. Using `ALTER TABLE my_table UPDATE` to update rows triggers a mutation, which causes parts to be re-written and uses IO/resources. With `ReplacingMergeTree`, you can simply insert the updated rows and the old rows will be replaced according to the table sorting key.
-
-## Creating a Table {#table_engine-mergetree-creating-a-table}
+## Creating Tables {#table_engine-mergetree-creating-a-table}
``` sql
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
(
- name1 [type1] [[NOT] NULL] [DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL expr1] [COMMENT ...] [CODEC(codec1)] [STATISTIC(stat1)] [TTL expr1] [PRIMARY KEY] [SETTINGS (name = value, ...)],
- name2 [type2] [[NOT] NULL] [DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL expr2] [COMMENT ...] [CODEC(codec2)] [STATISTIC(stat2)] [TTL expr2] [PRIMARY KEY] [SETTINGS (name = value, ...)],
+ name1 [type1] [[NOT] NULL] [DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL expr1] [COMMENT ...] [CODEC(codec1)] [STATISTICS(stat1)] [TTL expr1] [PRIMARY KEY] [SETTINGS (name = value, ...)],
+ name2 [type2] [[NOT] NULL] [DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL expr2] [COMMENT ...] [CODEC(codec2)] [STATISTICS(stat2)] [TTL expr2] [PRIMARY KEY] [SETTINGS (name = value, ...)],
...
INDEX index_name1 expr1 TYPE type1(...) [GRANULARITY value1],
INDEX index_name2 expr2 TYPE type2(...) [GRANULARITY value2],
@@ -59,23 +50,24 @@ ORDER BY expr
[SETTINGS name = value, ...]
```
-For a description of parameters, see the [CREATE query description](/docs/en/sql-reference/statements/create/table.md).
+For a detailed description of the parameters, see the [CREATE TABLE](/docs/en/sql-reference/statements/create/table.md) statement
### Query Clauses {#mergetree-query-clauses}
#### ENGINE
-`ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. The `MergeTree` engine does not have parameters.
+`ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. The `MergeTree` engine has no parameters.
#### ORDER_BY
`ORDER BY` — The sorting key.
-A tuple of column names or arbitrary expressions. Example: `ORDER BY (CounterID, EventDate)`.
+A tuple of column names or arbitrary expressions. Example: `ORDER BY (CounterID + 1, EventDate)`.
-ClickHouse uses the sorting key as a primary key if the primary key is not defined explicitly by the `PRIMARY KEY` clause.
+If no primary key is defined (i.e. `PRIMARY KEY` was not specified), ClickHouse uses the the sorting key as primary key.
-Use the `ORDER BY tuple()` syntax, if you do not need sorting, or set `create_table_empty_primary_key_by_default` to `true` to use the `ORDER BY tuple()` syntax by default. See [Selecting the Primary Key](#selecting-the-primary-key).
+If no sorting is required, you can use syntax `ORDER BY tuple()`.
+Alternatively, if setting `create_table_empty_primary_key_by_default` is enabled, `ORDER BY tuple()` is implicitly added to `CREATE TABLE` statements. See [Selecting a Primary Key](#selecting-a-primary-key).
#### PARTITION BY
@@ -87,100 +79,32 @@ For partitioning by month, use the `toYYYYMM(date_column)` expression, where `da
`PRIMARY KEY` — The primary key if it [differs from the sorting key](#choosing-a-primary-key-that-differs-from-the-sorting-key). Optional.
-By default the primary key is the same as the sorting key (which is specified by the `ORDER BY` clause). Thus in most cases it is unnecessary to specify a separate `PRIMARY KEY` clause.
+Specifying a sorting key (using `ORDER BY` clause) implicitly specifies a primary key.
+It is usually not necessary to specify the primary key in addition to the primary key.
#### SAMPLE BY
-`SAMPLE BY` — An expression for sampling. Optional.
+`SAMPLE BY` — A sampling expression. Optional.
-If a sampling expression is used, the primary key must contain it. The result of a sampling expression must be an unsigned integer. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`.
+If specified, it must be contained in the primary key.
+The sampling expression must result in an unsigned integer.
+
+Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`.
#### TTL
-`TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). Optional.
+`TTL` — A list of rules that specify the storage duration of rows and the logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). Optional.
-Expression must have one `Date` or `DateTime` column as a result. Example:
-```
-TTL date + INTERVAL 1 DAY
-```
+Expression must result in a `Date` or `DateTime`, e.g. `TTL date + INTERVAL 1 DAY`.
Type of the rule `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'|GROUP BY` specifies an action to be done with the part if the expression is satisfied (reaches current time): removal of expired rows, moving a part (if expression is satisfied for all rows in a part) to specified disk (`TO DISK 'xxx'`) or to volume (`TO VOLUME 'xxx'`), or aggregating values in expired rows. Default type of the rule is removal (`DELETE`). List of multiple rules can be specified, but there should be no more than one `DELETE` rule.
+
For more details, see [TTL for columns and tables](#table_engine-mergetree-ttl)
-### SETTINGS
-Additional parameters that control the behavior of the `MergeTree` (optional):
+#### SETTINGS
-#### index_granularity
-
-`index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [Data Storage](#mergetree-data-storage).
-
-#### index_granularity_bytes
-
-`index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [Data Storage](#mergetree-data-storage).
-
-#### min_index_granularity_bytes
-
-`min_index_granularity_bytes` — Min allowed size of data granules in bytes. Default value: 1024b. To provide a safeguard against accidentally creating tables with very low index_granularity_bytes. See [Data Storage](#mergetree-data-storage).
-
-#### enable_mixed_granularity_parts
-
-`enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` setting. Before version 19.11, there was only the `index_granularity` setting for restricting granule size. The `index_granularity_bytes` setting improves ClickHouse performance when selecting data from tables with big rows (tens and hundreds of megabytes). If you have tables with big rows, you can enable this setting for the tables to improve the efficiency of `SELECT` queries.
-
-#### use_minimalistic_part_header_in_zookeeper
-
-`use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, then ZooKeeper stores less data. For more information, see the [setting description](/docs/en/operations/server-configuration-parameters/settings.md/#server-settings-use_minimalistic_part_header_in_zookeeper) in “Server configuration parameters”.
-
-#### min_merge_bytes_to_use_direct_io
-
-`min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, ClickHouse reads and writes the data to the storage disk using the direct I/O interface (`O_DIRECT` option). If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes.
-
-#### merge_with_ttl_timeout
-
-`merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with delete TTL. Default value: `14400` seconds (4 hours).
-#### merge_with_recompression_ttl_timeout
-
-`merge_with_recompression_ttl_timeout` — Minimum delay in seconds before repeating a merge with recompression TTL. Default value: `14400` seconds (4 hours).
-
-#### try_fetch_recompressed_part_timeout
-
-`try_fetch_recompressed_part_timeout` — Timeout (in seconds) before starting merge with recompression. During this time ClickHouse tries to fetch recompressed part from replica which assigned this merge with recompression. Default value: `7200` seconds (2 hours).
-
-#### write_final_mark
-
-`write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don’t turn it off.
-
-#### merge_max_block_size
-
-`merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192.
-
-#### storage_policy
-
-`storage_policy` — Storage policy. See [Using Multiple Block Devices for Data Storage](#table_engine-mergetree-multiple-volumes).
-
-#### min_bytes_for_wide_part
-
-`min_bytes_for_wide_part`, `min_rows_for_wide_part` — Minimum number of bytes/rows in a data part that can be stored in `Wide` format. You can set one, both or none of these settings. See [Data Storage](#mergetree-data-storage).
-
-#### max_parts_in_total
-
-`max_parts_in_total` — Maximum number of parts in all partitions.
-
-#### max_compress_block_size
-
-`max_compress_block_size` — Maximum size of blocks of uncompressed data before compressing for writing to a table. You can also specify this setting in the global settings (see [max_compress_block_size](/docs/en/operations/settings/settings.md/#max-compress-block-size) setting). The value specified when table is created overrides the global value for this setting.
-
-#### min_compress_block_size
-
-`min_compress_block_size` — Minimum size of blocks of uncompressed data required for compression when writing the next mark. You can also specify this setting in the global settings (see [min_compress_block_size](/docs/en/operations/settings/settings.md/#min-compress-block-size) setting). The value specified when table is created overrides the global value for this setting.
-
-#### max_partitions_to_read
-
-`max_partitions_to_read` — Limits the maximum number of partitions that can be accessed in one query. You can also specify setting [max_partitions_to_read](/docs/en/operations/settings/merge-tree-settings.md/#max-partitions-to-read) in the global setting.
-
-#### allow_experimental_optimized_row_order
-
-`allow_experimental_optimized_row_order` - Experimental. Enables the optimization of the row order during inserts to improve the compressability of the data for compression codecs (e.g. LZ4). Analyzes and reorders the data, and thus increases the CPU overhead of inserts.
+See [MergeTree Settings](../../../operations/settings/merge-tree-settings.md).
**Example of Sections Setting**
@@ -270,7 +194,7 @@ ClickHouse does not require a unique primary key. You can insert multiple rows w
You can use `Nullable`-typed expressions in the `PRIMARY KEY` and `ORDER BY` clauses but it is strongly discouraged. To allow this feature, turn on the [allow_nullable_key](/docs/en/operations/settings/settings.md/#allow-nullable-key) setting. The [NULLS_LAST](/docs/en/sql-reference/statements/select/order-by.md/#sorting-of-special-values) principle applies for `NULL` values in the `ORDER BY` clause.
-### Selecting the Primary Key {#selecting-the-primary-key}
+### Selecting a Primary Key {#selecting-a-primary-key}
The number of columns in the primary key is not explicitly limited. Depending on the data structure, you can include more or fewer columns in the primary key. This may:
@@ -1043,12 +967,12 @@ ClickHouse versions 22.3 through 22.7 use a different cache configuration, see [
## Column Statistics (Experimental) {#column-statistics}
-The statistic declaration is in the columns section of the `CREATE` query for tables from the `*MergeTree*` Family when we enable `set allow_experimental_statistic = 1`.
+The statistics declaration is in the columns section of the `CREATE` query for tables from the `*MergeTree*` Family when we enable `set allow_experimental_statistics = 1`.
``` sql
CREATE TABLE tab
(
- a Int64 STATISTIC(tdigest),
+ a Int64 STATISTICS(TDigest, Uniq),
b Float64
)
ENGINE = MergeTree
@@ -1058,19 +982,23 @@ ORDER BY a
We can also manipulate statistics with `ALTER` statements.
```sql
-ALTER TABLE tab ADD STATISTIC b TYPE tdigest;
-ALTER TABLE tab DROP STATISTIC a TYPE tdigest;
+ALTER TABLE tab ADD STATISTICS b TYPE TDigest, Uniq;
+ALTER TABLE tab DROP STATISTICS a;
```
-These lightweight statistics aggregate information about distribution of values in columns.
-They can be used for query optimization when we enable `set allow_statistic_optimize = 1`.
+These lightweight statistics aggregate information about distribution of values in columns. Statistics are stored in every part and updated when every insert comes.
+They can be used for prewhere optimization only if we enable `set allow_statistics_optimize = 1`.
#### Available Types of Column Statistics {#available-types-of-column-statistics}
-- `tdigest`
+- `TDigest`
Stores distribution of values from numeric columns in [TDigest](https://github.com/tdunning/t-digest) sketch.
+- `Uniq`
+
+ Estimate the number of distinct values of a column by HyperLogLog.
+
## Column-level Settings {#column-level-settings}
Certain MergeTree settings can be override at column level:
diff --git a/docs/en/engines/table-engines/special/file.md b/docs/en/engines/table-engines/special/file.md
index 0d422f64762..957b18b5305 100644
--- a/docs/en/engines/table-engines/special/file.md
+++ b/docs/en/engines/table-engines/special/file.md
@@ -102,6 +102,7 @@ For partitioning by month, use the `toYYYYMM(date_column)` expression, where `da
- `_path` — Path to the file. Type: `LowCardinalty(String)`.
- `_file` — Name of the file. Type: `LowCardinalty(String)`.
- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the size is unknown, the value is `NULL`.
+- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
## Settings {#settings}
diff --git a/docs/en/engines/table-engines/special/url.md b/docs/en/engines/table-engines/special/url.md
index f6183a779ae..c906830d0e9 100644
--- a/docs/en/engines/table-engines/special/url.md
+++ b/docs/en/engines/table-engines/special/url.md
@@ -108,6 +108,7 @@ For partitioning by month, use the `toYYYYMM(date_column)` expression, where `da
- `_path` — Path to the `URL`. Type: `LowCardinalty(String)`.
- `_file` — Resource name of the `URL`. Type: `LowCardinalty(String)`.
- `_size` — Size of the resource in bytes. Type: `Nullable(UInt64)`. If the size is unknown, the value is `NULL`.
+- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
## Storage Settings {#storage-settings}
diff --git a/docs/en/getting-started/example-datasets/images/stackoverflow.png b/docs/en/getting-started/example-datasets/images/stackoverflow.png
new file mode 100644
index 00000000000..f31acdc8cc3
Binary files /dev/null and b/docs/en/getting-started/example-datasets/images/stackoverflow.png differ
diff --git a/docs/en/getting-started/example-datasets/stackoverflow.md b/docs/en/getting-started/example-datasets/stackoverflow.md
new file mode 100644
index 00000000000..e982a3c3dfc
--- /dev/null
+++ b/docs/en/getting-started/example-datasets/stackoverflow.md
@@ -0,0 +1,394 @@
+---
+slug: /en/getting-started/example-datasets/stackoverflow
+sidebar_label: Stack Overflow
+sidebar_position: 1
+description: Analyzing Stack Overflow data with ClickHouse
+---
+
+# Analyzing Stack Overflow data with ClickHouse
+
+This dataset contains every `Post`, `User`, `Vote`, `Comment`, `Badge, `PostHistory`, and `PostLink` that has occurred on Stack Overflow.
+
+Users can either download pre-prepared Parquet versions of the data, containing every post up to April 2024, or download the latest data in XML format and load this. Stack Overflow provide updates to this data periodically - historically every 3 months.
+
+The following diagram shows the schema for the available tables assuming Parquet format.
+
+
+
+A description of the schema of this data can be found [here](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede).
+
+## Pre-prepared data
+
+We provide a copy of this data in Parquet format, up to date as of April 2024. While small for ClickHouse with respect to the number of rows (60 million posts), this dataset contains significant volumes of text and large String columns.
+
+```sql
+CREATE DATABASE stackoverflow
+```
+
+The following timings are for a 96 GiB, 24 vCPU ClickHouse Cloud cluster located in `eu-west-2`. The dataset is located in `eu-west-3`.
+
+### Posts
+
+```sql
+CREATE TABLE stackoverflow.posts
+(
+ `Id` Int32 CODEC(Delta(4), ZSTD(1)),
+ `PostTypeId` Enum8('Question' = 1, 'Answer' = 2, 'Wiki' = 3, 'TagWikiExcerpt' = 4, 'TagWiki' = 5, 'ModeratorNomination' = 6, 'WikiPlaceholder' = 7, 'PrivilegeWiki' = 8),
+ `AcceptedAnswerId` UInt32,
+ `CreationDate` DateTime64(3, 'UTC'),
+ `Score` Int32,
+ `ViewCount` UInt32 CODEC(Delta(4), ZSTD(1)),
+ `Body` String,
+ `OwnerUserId` Int32,
+ `OwnerDisplayName` String,
+ `LastEditorUserId` Int32,
+ `LastEditorDisplayName` String,
+ `LastEditDate` DateTime64(3, 'UTC') CODEC(Delta(8), ZSTD(1)),
+ `LastActivityDate` DateTime64(3, 'UTC'),
+ `Title` String,
+ `Tags` String,
+ `AnswerCount` UInt16 CODEC(Delta(2), ZSTD(1)),
+ `CommentCount` UInt8,
+ `FavoriteCount` UInt8,
+ `ContentLicense` LowCardinality(String),
+ `ParentId` String,
+ `CommunityOwnedDate` DateTime64(3, 'UTC'),
+ `ClosedDate` DateTime64(3, 'UTC')
+)
+ENGINE = MergeTree
+PARTITION BY toYear(CreationDate)
+ORDER BY (PostTypeId, toDate(CreationDate), CreationDate)
+
+INSERT INTO stackoverflow.posts SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/*.parquet')
+
+0 rows in set. Elapsed: 265.466 sec. Processed 59.82 million rows, 38.07 GB (225.34 thousand rows/s., 143.42 MB/s.)
+```
+
+Posts are also available by year e.g. [https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet](https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet)
+
+
+### Votes
+
+```sql
+CREATE TABLE stackoverflow.votes
+(
+ `Id` UInt32,
+ `PostId` Int32,
+ `VoteTypeId` UInt8,
+ `CreationDate` DateTime64(3, 'UTC'),
+ `UserId` Int32,
+ `BountyAmount` UInt8
+)
+ENGINE = MergeTree
+ORDER BY (VoteTypeId, CreationDate, PostId, UserId)
+
+INSERT INTO stackoverflow.votes SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/votes/*.parquet')
+
+0 rows in set. Elapsed: 21.605 sec. Processed 238.98 million rows, 2.13 GB (11.06 million rows/s., 98.46 MB/s.)
+```
+
+Votes are also available by year e.g. [https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet](https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/votes/2020.parquet)
+
+
+### Comments
+
+```sql
+CREATE TABLE stackoverflow.comments
+(
+ `Id` UInt32,
+ `PostId` UInt32,
+ `Score` UInt16,
+ `Text` String,
+ `CreationDate` DateTime64(3, 'UTC'),
+ `UserId` Int32,
+ `UserDisplayName` LowCardinality(String)
+)
+ENGINE = MergeTree
+ORDER BY CreationDate
+
+INSERT INTO stackoverflow.comments SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/comments/*.parquet')
+
+0 rows in set. Elapsed: 56.593 sec. Processed 90.38 million rows, 11.14 GB (1.60 million rows/s., 196.78 MB/s.)
+```
+
+Comments are also available by year e.g. [https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet](https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/comments/2020.parquet)
+
+### Users
+
+```sql
+CREATE TABLE stackoverflow.users
+(
+ `Id` Int32,
+ `Reputation` LowCardinality(String),
+ `CreationDate` DateTime64(3, 'UTC') CODEC(Delta(8), ZSTD(1)),
+ `DisplayName` String,
+ `LastAccessDate` DateTime64(3, 'UTC'),
+ `AboutMe` String,
+ `Views` UInt32,
+ `UpVotes` UInt32,
+ `DownVotes` UInt32,
+ `WebsiteUrl` String,
+ `Location` LowCardinality(String),
+ `AccountId` Int32
+)
+ENGINE = MergeTree
+ORDER BY (Id, CreationDate)
+
+INSERT INTO stackoverflow.users SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/users.parquet')
+
+0 rows in set. Elapsed: 10.988 sec. Processed 22.48 million rows, 1.36 GB (2.05 million rows/s., 124.10 MB/s.)
+```
+
+### Badges
+
+```sql
+CREATE TABLE stackoverflow.badges
+(
+ `Id` UInt32,
+ `UserId` Int32,
+ `Name` LowCardinality(String),
+ `Date` DateTime64(3, 'UTC'),
+ `Class` Enum8('Gold' = 1, 'Silver' = 2, 'Bronze' = 3),
+ `TagBased` Bool
+)
+ENGINE = MergeTree
+ORDER BY UserId
+
+INSERT INTO stackoverflow.badges SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/badges.parquet')
+
+0 rows in set. Elapsed: 6.635 sec. Processed 51.29 million rows, 797.05 MB (7.73 million rows/s., 120.13 MB/s.)
+```
+
+### `PostLinks`
+
+```sql
+CREATE TABLE stackoverflow.postlinks
+(
+ `Id` UInt64,
+ `CreationDate` DateTime64(3, 'UTC'),
+ `PostId` Int32,
+ `RelatedPostId` Int32,
+ `LinkTypeId` Enum8('Linked' = 1, 'Duplicate' = 3)
+)
+ENGINE = MergeTree
+ORDER BY (PostId, RelatedPostId)
+
+INSERT INTO stackoverflow.postlinks SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/postlinks.parquet')
+
+0 rows in set. Elapsed: 1.534 sec. Processed 6.55 million rows, 129.70 MB (4.27 million rows/s., 84.57 MB/s.)
+```
+
+### `PostHistory`
+
+```sql
+CREATE TABLE stackoverflow.posthistory
+(
+ `Id` UInt64,
+ `PostHistoryTypeId` UInt8,
+ `PostId` Int32,
+ `RevisionGUID` String,
+ `CreationDate` DateTime64(3, 'UTC'),
+ `UserId` Int32,
+ `Text` String,
+ `ContentLicense` LowCardinality(String),
+ `Comment` String,
+ `UserDisplayName` String
+)
+ENGINE = MergeTree
+ORDER BY (CreationDate, PostId)
+
+INSERT INTO stackoverflow.posthistory SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posthistory/*.parquet')
+
+0 rows in set. Elapsed: 422.795 sec. Processed 160.79 million rows, 67.08 GB (380.30 thousand rows/s., 158.67 MB/s.)
+```
+
+## Original dataset
+
+The original dataset is available in compressed (7zip) XML format at [https://archive.org/download/stackexchange](https://archive.org/download/stackexchange) - files with prefix `stackoverflow.com*`.
+
+### Download
+
+```bash
+wget https://archive.org/download/stackexchange/stackoverflow.com-Badges.7z
+wget https://archive.org/download/stackexchange/stackoverflow.com-Comments.7z
+wget https://archive.org/download/stackexchange/stackoverflow.com-PostHistory.7z
+wget https://archive.org/download/stackexchange/stackoverflow.com-PostLinks.7z
+wget https://archive.org/download/stackexchange/stackoverflow.com-Posts.7z
+wget https://archive.org/download/stackexchange/stackoverflow.com-Users.7z
+wget https://archive.org/download/stackexchange/stackoverflow.com-Votes.7z
+```
+
+These files are up to 35GB and can take around 30 mins to download depending on internet connection - the download server throttles at around 20MB/sec.
+
+### Convert to JSON
+
+At the time of writing, ClickHouse does not have native support for XML as an input format. To load the data into ClickHouse we first convert to NDJSON.
+
+To convert XML to JSON we recommend the [`xq`](https://github.com/kislyuk/yq) linux tool, a simple `jq` wrapper for XML documents.
+
+Install xq and jq:
+
+```bash
+sudo apt install jq
+pip install yq
+```
+
+The following steps apply to any of the above files. We use the `stackoverflow.com-Posts.7z` file as an example. Modify as required.
+
+Extract the file using [p7zip](https://p7zip.sourceforge.net/). This will produce a single xml file - in this case `Posts.xml`.
+
+> Files are compressed approximately 4.5x. At 22GB compressed, the posts file requires around 97G uncompressed.
+
+```bash
+p7zip -d stackoverflow.com-Posts.7z
+```
+
+The following splits the xml file into files, each containing 10000 rows.
+
+```bash
+mkdir posts
+cd posts
+# the following splits the input xml file into sub files of 10000 rows
+tail +3 ../Posts.xml | head -n -1 | split -l 10000 --filter='{ printf "\n"; cat - ; printf "\n"; } > $FILE' -
+```
+
+After running the above users will have a set of files, each with 10000 lines. This ensures the memory overhead of the next command is not excessive (xml to JSON conversion is done in memory).
+
+```bash
+find . -maxdepth 1 -type f -exec xq -c '.rows.row[]' {} \; | sed -e 's:"@:":g' > posts_v2.json
+```
+
+The above command will produce a single `posts.json` file.
+
+Load into ClickHouse with the following command. Note the schema is specified for the `posts.json` file. This will need to be adjusted per data type to align with the target table.
+
+```bash
+clickhouse local --query "SELECT * FROM file('posts.json', JSONEachRow, 'Id Int32, PostTypeId UInt8, AcceptedAnswerId UInt32, CreationDate DateTime64(3, \'UTC\'), Score Int32, ViewCount UInt32, Body String, OwnerUserId Int32, OwnerDisplayName String, LastEditorUserId Int32, LastEditorDisplayName String, LastEditDate DateTime64(3, \'UTC\'), LastActivityDate DateTime64(3, \'UTC\'), Title String, Tags String, AnswerCount UInt16, CommentCount UInt8, FavoriteCount UInt8, ContentLicense String, ParentId String, CommunityOwnedDate DateTime64(3, \'UTC\'), ClosedDate DateTime64(3, \'UTC\')') FORMAT Native" | clickhouse client --host --secure --password --query "INSERT INTO stackoverflow.posts_v2 FORMAT Native"
+```
+
+## Example queries
+
+A few simple questions to you get started.
+
+### Most popular tags on Stack Overflow
+
+```sql
+
+SELECT
+ arrayJoin(arrayFilter(t -> (t != ''), splitByChar('|', Tags))) AS Tags,
+ count() AS c
+FROM stackoverflow.posts
+GROUP BY Tags
+ORDER BY c DESC
+LIMIT 10
+
+┌─Tags───────┬───────c─┐
+│ javascript │ 2527130 │
+│ python │ 2189638 │
+│ java │ 1916156 │
+│ c# │ 1614236 │
+│ php │ 1463901 │
+│ android │ 1416442 │
+│ html │ 1186567 │
+│ jquery │ 1034621 │
+│ c++ │ 806202 │
+│ css │ 803755 │
+└────────────┴─────────┘
+
+10 rows in set. Elapsed: 1.013 sec. Processed 59.82 million rows, 1.21 GB (59.07 million rows/s., 1.19 GB/s.)
+Peak memory usage: 224.03 MiB.
+```
+
+### User with the most answers (active accounts)
+
+Account requires a `UserId`.
+
+```sql
+SELECT
+ any(OwnerUserId) UserId,
+ OwnerDisplayName,
+ count() AS c
+FROM stackoverflow.posts WHERE OwnerDisplayName != '' AND PostTypeId='Answer' AND OwnerUserId != 0
+GROUP BY OwnerDisplayName
+ORDER BY c DESC
+LIMIT 5
+
+┌─UserId─┬─OwnerDisplayName─┬────c─┐
+│ 22656 │ Jon Skeet │ 2727 │
+│ 23354 │ Marc Gravell │ 2150 │
+│ 12950 │ tvanfosson │ 1530 │
+│ 3043 │ Joel Coehoorn │ 1438 │
+│ 10661 │ S.Lott │ 1087 │
+└────────┴──────────────────┴──────┘
+
+5 rows in set. Elapsed: 0.154 sec. Processed 35.83 million rows, 193.39 MB (232.33 million rows/s., 1.25 GB/s.)
+Peak memory usage: 206.45 MiB.
+```
+
+### ClickHouse related posts with the most views
+
+```sql
+SELECT
+ Id,
+ Title,
+ ViewCount,
+ AnswerCount
+FROM stackoverflow.posts
+WHERE Title ILIKE '%ClickHouse%'
+ORDER BY ViewCount DESC
+LIMIT 10
+
+┌───────Id─┬─Title────────────────────────────────────────────────────────────────────────────┬─ViewCount─┬─AnswerCount─┐
+│ 52355143 │ Is it possible to delete old records from clickhouse table? │ 41462 │ 3 │
+│ 37954203 │ Clickhouse Data Import │ 38735 │ 3 │
+│ 37901642 │ Updating data in Clickhouse │ 36236 │ 6 │
+│ 58422110 │ Pandas: How to insert dataframe into Clickhouse │ 29731 │ 4 │
+│ 63621318 │ DBeaver - Clickhouse - SQL Error [159] .. Read timed out │ 27350 │ 1 │
+│ 47591813 │ How to filter clickhouse table by array column contents? │ 27078 │ 2 │
+│ 58728436 │ How to search the string in query with case insensitive on Clickhouse database? │ 26567 │ 3 │
+│ 65316905 │ Clickhouse: DB::Exception: Memory limit (for query) exceeded │ 24899 │ 2 │
+│ 49944865 │ How to add a column in clickhouse │ 24424 │ 1 │
+│ 59712399 │ How to cast date Strings to DateTime format with extended parsing in ClickHouse? │ 22620 │ 1 │
+└──────────┴──────────────────────────────────────────────────────────────────────────────────┴───────────┴─────────────┘
+
+10 rows in set. Elapsed: 0.472 sec. Processed 59.82 million rows, 1.91 GB (126.63 million rows/s., 4.03 GB/s.)
+Peak memory usage: 240.01 MiB.
+```
+
+### Most controversial posts
+
+```sql
+SELECT
+ Id,
+ Title,
+ UpVotes,
+ DownVotes,
+ abs(UpVotes - DownVotes) AS Controversial_ratio
+FROM stackoverflow.posts
+INNER JOIN
+(
+ SELECT
+ PostId,
+ countIf(VoteTypeId = 2) AS UpVotes,
+ countIf(VoteTypeId = 3) AS DownVotes
+ FROM stackoverflow.votes
+ GROUP BY PostId
+ HAVING (UpVotes > 10) AND (DownVotes > 10)
+) AS votes ON posts.Id = votes.PostId
+WHERE Title != ''
+ORDER BY Controversial_ratio ASC
+LIMIT 3
+
+┌───────Id─┬─Title─────────────────────────────────────────────┬─UpVotes─┬─DownVotes─┬─Controversial_ratio─┐
+│ 583177 │ VB.NET Infinite For Loop │ 12 │ 12 │ 0 │
+│ 9756797 │ Read console input as enumerable - one statement? │ 16 │ 16 │ 0 │
+│ 13329132 │ What's the point of ARGV in Ruby? │ 22 │ 22 │ 0 │
+└──────────┴───────────────────────────────────────────────────┴─────────┴───────────┴─────────────────────┘
+
+3 rows in set. Elapsed: 4.779 sec. Processed 298.80 million rows, 3.16 GB (62.52 million rows/s., 661.05 MB/s.)
+Peak memory usage: 6.05 GiB.
+```
+
+## Attribution
+
+We thank Stack Overflow for providing this data under the `cc-by-sa 4.0` license, acknowledging their efforts and the original source of the data at [https://archive.org/details/stackexchange](https://archive.org/details/stackexchange).
diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md
index 67752f223ce..98e73dec451 100644
--- a/docs/en/getting-started/install.md
+++ b/docs/en/getting-started/install.md
@@ -314,7 +314,7 @@ For example, to download a aarch64 binary for ClickHouse v23.4, follow these ste
- Find the GitHub pull request for release v23.4: [Release pull request for branch 23.4](https://github.com/ClickHouse/ClickHouse/pull/49238)
- Click "Commits", then click a commit similar to "Update autogenerated version to 23.4.2.1 and contributors" for the particular version you like to install.
- Click the green check / yellow dot / red cross to open the list of CI checks.
-- Click "Details" next to "ClickHouse Build Check" in the list, it will open a page similar to [this page](https://s3.amazonaws.com/clickhouse-test-reports/46793/b460eb70bf29b19eadd19a1f959b15d186705394/clickhouse_build_check/report.html)
+- Click "Details" next to "Builds" in the list, it will open a page similar to [this page](https://s3.amazonaws.com/clickhouse-test-reports/46793/b460eb70bf29b19eadd19a1f959b15d186705394/clickhouse_build_check/report.html)
- Find the rows with compiler = "clang-*-aarch64" - there are multiple rows.
- Download the artifacts for these builds.
diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md
index 1eb426af617..e18ff6f1a3f 100644
--- a/docs/en/interfaces/cli.md
+++ b/docs/en/interfaces/cli.md
@@ -193,6 +193,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
- `--hardware-utilization` — Print hardware utilization information in progress bar.
- `--print-profile-events` – Print `ProfileEvents` packets.
- `--profile-events-delay-ms` – Delay between printing `ProfileEvents` packets (-1 - print only totals, 0 - print every single packet).
+- `--jwt` – If specified, enables authorization via JSON Web Token. Server JWT authorization is available only in ClickHouse Cloud.
Instead of `--host`, `--port`, `--user` and `--password` options, ClickHouse client also supports connection strings (see next section).
diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index 66d5bd2e574..a81a17e65d6 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -480,7 +480,7 @@ The CSV format supports the output of totals and extremes the same way as `TabSe
- [input_format_csv_detect_header](/docs/en/operations/settings/settings-formats.md/#input_format_csv_detect_header) - automatically detect header with names and types in CSV format. Default value - `true`.
- [input_format_csv_skip_trailing_empty_lines](/docs/en/operations/settings/settings-formats.md/#input_format_csv_skip_trailing_empty_lines) - skip trailing empty lines at the end of data. Default value - `false`.
- [input_format_csv_trim_whitespaces](/docs/en/operations/settings/settings-formats.md/#input_format_csv_trim_whitespaces) - trim spaces and tabs in non-quoted CSV strings. Default value - `true`.
-- [input_format_csv_allow_whitespace_or_tab_as_delimiter](/docs/en/operations/settings/settings-formats.md/# input_format_csv_allow_whitespace_or_tab_as_delimiter) - Allow to use whitespace or tab as field delimiter in CSV strings. Default value - `false`.
+- [input_format_csv_allow_whitespace_or_tab_as_delimiter](/docs/en/operations/settings/settings-formats.md/#input_format_csv_allow_whitespace_or_tab_as_delimiter) - Allow to use whitespace or tab as field delimiter in CSV strings. Default value - `false`.
- [input_format_csv_allow_variable_number_of_columns](/docs/en/operations/settings/settings-formats.md/#input_format_csv_allow_variable_number_of_columns) - allow variable number of columns in CSV format, ignore extra columns and use default values on missing columns. Default value - `false`.
- [input_format_csv_use_default_on_bad_values](/docs/en/operations/settings/settings-formats.md/#input_format_csv_use_default_on_bad_values) - Allow to set default value to column when CSV field deserialization failed on bad value. Default value - `false`.
- [input_format_csv_try_infer_numbers_from_strings](/docs/en/operations/settings/settings-formats.md/#input_format_csv_try_infer_numbers_from_strings) - Try to infer numbers from string fields while schema inference. Default value - `false`.
@@ -1490,6 +1490,8 @@ Differs from [PrettySpaceNoEscapes](#prettyspacenoescapes) in that up to 10,000
- [output_format_pretty_color](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_color) - use ANSI escape sequences to paint colors in Pretty formats. Default value - `true`.
- [output_format_pretty_grid_charset](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_grid_charset) - Charset for printing grid borders. Available charsets: ASCII, UTF-8. Default value - `UTF-8`.
- [output_format_pretty_row_numbers](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_row_numbers) - Add row numbers before each row for pretty output format. Default value - `true`.
+- [output_format_pretty_display_footer_column_names](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_display_footer_column_names) - Display column names in the footer if table contains many rows. Default value - `true`.
+- [output_format_pretty_display_footer_column_names_min_rows](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_display_footer_column_names_min_rows) - Sets the minimum number of rows for which a footer will be displayed if [output_format_pretty_display_footer_column_names](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_display_footer_column_names) is enabled. Default value - 50.
## RowBinary {#rowbinary}
@@ -2165,6 +2167,9 @@ To exchange data with Hadoop, you can use [HDFS table engine](/docs/en/engines/t
- [output_format_parquet_fixed_string_as_fixed_byte_array](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_fixed_string_as_fixed_byte_array) - use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary/String for FixedString columns. Default value - `true`.
- [output_format_parquet_version](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_version) - The version of Parquet format used in output format. Default value - `2.latest`.
- [output_format_parquet_compression_method](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_compression_method) - compression method used in output Parquet format. Default value - `lz4`.
+- [input_format_parquet_max_block_size](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_max_block_size) - Max block row size for parquet reader. Default value - `65409`.
+- [input_format_parquet_prefer_block_bytes](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_prefer_block_bytes) - Average block bytes output by parquet reader. Default value - `16744704`.
+- [output_format_parquet_write_page_index](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_max_block_size) - Add a possibility to write page index into parquet files. Need to disable `output_format_parquet_use_custom_encoder` at present. Default value - `true`.
## ParquetMetadata {data-format-parquet-metadata}
diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md
index eb1a3ba1dbc..f5b6326fa96 100644
--- a/docs/en/interfaces/http.md
+++ b/docs/en/interfaces/http.md
@@ -508,7 +508,7 @@ Now `rule` can configure `method`, `headers`, `url`, `handler`:
- `headers` are responsible for matching the header part of the HTTP request. It is compatible with RE2’s regular expressions. It is an optional configuration. If it is not defined in the configuration file, it does not match the header portion of the HTTP request.
-- `handler` contains the main processing part. Now `handler` can configure `type`, `status`, `content_type`, `response_content`, `query`, `query_param_name`.
+- `handler` contains the main processing part. Now `handler` can configure `type`, `status`, `content_type`, `http_response_headers`, `response_content`, `query`, `query_param_name`.
`type` currently supports three types: [predefined_query_handler](#predefined_query_handler), [dynamic_query_handler](#dynamic_query_handler), [static](#static).
- `query` — use with `predefined_query_handler` type, executes query when the handler is called.
@@ -519,6 +519,8 @@ Now `rule` can configure `method`, `headers`, `url`, `handler`:
- `content_type` — use with any type, response [content-type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type).
+ - `http_response_headers` — use with any type, response headers map. Could be used to set content type as well.
+
- `response_content` — use with `static` type, response content sent to client, when using the prefix ‘file://’ or ‘config://’, find the content from the file or configuration sends to client.
Next are the configuration methods for different `type`.
@@ -616,6 +618,33 @@ Return a message.
static
402
text/html; charset=UTF-8
+
+ en
+ 43
+
+ Say Hi!
+
+
+
+
+```
+
+`http_response_headers` could be used to set content type instead of `content_type`.
+
+``` xml
+
+
+ GET
+ xxx
+ /hi
+
+ static
+ 402
+
+ text/html; charset=UTF-8
+ en
+ 43
+
Say Hi!
@@ -696,6 +725,9 @@ Find the content from the file send to client.
static
text/html; charset=UTF-8
+
+ 737060cd8c284d8af7ad3082f209582d
+
file:///absolute_path_file.html
@@ -706,6 +738,9 @@ Find the content from the file send to client.
static
text/html; charset=UTF-8
+
+ 737060cd8c284d8af7ad3082f209582d
+
file://./relative_path_file.html
diff --git a/docs/en/interfaces/mysql.md b/docs/en/interfaces/mysql.md
index ce5ab24ecb0..42820505406 100644
--- a/docs/en/interfaces/mysql.md
+++ b/docs/en/interfaces/mysql.md
@@ -31,6 +31,56 @@ Alternatively, in order to enable the MySQL interface for an existing service:
3. After entering the password, you will get prompted the MySQL connection string for this service
+## Creating multiple MySQL users in ClickHouse Cloud
+
+By default, there is a built-in `mysql4` user, which uses the same password as the `default` one. The `` part is the first segment of your ClickHouse Cloud hostname. This format is necessary to work with the tools that implement secure connection, but don't provide [SNI information in their TLS handshake](https://www.cloudflare.com/learning/ssl/what-is-sni), which makes it impossible to do the internal routing without an extra hint in the username (MySQL console client is one of such tools).
+
+Because of this, we _highly recommend_ following the `mysql4_` format when creating a new user intended to be used with the MySQL interface, where `` is a hint to identify your Cloud service, and `` is an arbitrary suffix of your choice.
+
+:::tip
+For ClickHouse Cloud hostname like `foobar.us-east1.aws.clickhouse.cloud`, the `` part equals to `foobar`, and a custom MySQL username could look like `mysql4foobar_team1`.
+:::
+
+You can create extra users to use with the MySQL interface if, for example, you need to apply extra settings.
+
+1. Optional - create a [settings profile](https://clickhouse.com/docs/en/sql-reference/statements/create/settings-profile) to apply for your custom user. For example, `my_custom_profile` with an extra setting which will be applied by default when we connect with the user we create later:
+
+ ```sql
+ CREATE SETTINGS PROFILE my_custom_profile SETTINGS prefer_column_name_to_alias=1;
+ ```
+
+ `prefer_column_name_to_alias` is used just as an example, you can use other settings there.
+2. [Create a user](https://clickhouse.com/docs/en/sql-reference/statements/create/user) using the following format: `mysql4_` ([see above](#creating-multiple-mysql-users-in-clickhouse-cloud)). The password must be in double SHA1 format. For example:
+
+ ```sql
+ CREATE USER mysql4foobar_team1 IDENTIFIED WITH double_sha1_password BY 'YourPassword42$';
+ ```
+
+ or if you want to use a custom profile for this user:
+
+ ```sql
+ CREATE USER mysql4foobar_team1 IDENTIFIED WITH double_sha1_password BY 'YourPassword42$' SETTINGS PROFILE 'my_custom_profile';
+ ```
+
+ where `my_custom_profile` is the name of the profile you created earlier.
+3. [Grant](https://clickhouse.com/docs/en/sql-reference/statements/grant) the new user the necessary permissions to interact with the desired tables or databases. For example, if you want to grant access to `system.query_log` only:
+
+ ```sql
+ GRANT SELECT ON system.query_log TO mysql4foobar_team1;
+ ```
+
+4. Use the created user to connect to your ClickHouse Cloud service with the MySQL interface.
+
+### Troubleshooting multiple MySQL users in ClickHouse Cloud
+
+If you created a new MySQL user, and you see the following error while connecting via MySQL CLI client:
+
+```
+ERROR 2013 (HY000): Lost connection to MySQL server at 'reading authorization packet', system error: 54
+```
+
+In this case, ensure that the username follows the `mysql4_` format, as described ([above](#creating-multiple-mysql-users-in-clickhouse-cloud)).
+
## Enabling the MySQL Interface On Self-managed ClickHouse
Add the [mysql_port](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) setting to your server's configuration file. For example, you could define the port in a new XML file in your `config.d/` [folder](../operations/configuration-files):
diff --git a/docs/en/operations/allocation-profiling.md b/docs/en/operations/allocation-profiling.md
index 64b4106a7e1..574e1ae2ff3 100644
--- a/docs/en/operations/allocation-profiling.md
+++ b/docs/en/operations/allocation-profiling.md
@@ -59,10 +59,10 @@ For that, we need to use `jemalloc`'s tool called [jeprof](https://github.com/je
If that’s the case, we recommend installing an [alternative implementation](https://github.com/gimli-rs/addr2line) of the tool.
```
-git clone https://github.com/gimli-rs/addr2line
+git clone https://github.com/gimli-rs/addr2line.git --depth=1 --branch=0.23.0
cd addr2line
-cargo b --examples -r
-cp ./target/release/examples/addr2line path/to/current/addr2line
+cargo build --features bin --release
+cp ./target/release/addr2line path/to/current/addr2line
```
:::
diff --git a/docs/en/operations/analyzer.md b/docs/en/operations/analyzer.md
new file mode 100644
index 00000000000..298c6dacd06
--- /dev/null
+++ b/docs/en/operations/analyzer.md
@@ -0,0 +1,194 @@
+---
+slug: /en/operations/analyzer
+sidebar_label: Analyzer
+title: Analyzer
+description: Details about ClickHouse's query analyzer
+keywords: [analyzer]
+---
+
+# Analyzer
+
+
+
+## Known incompatibilities
+
+In ClickHouse version `24.3`, the new query analyzer was enabled by default.
+Despite fixing a large number of bugs and introducing new optimizations, it also introduces some breaking changes in ClickHouse behaviour. Please read the following changes to determine how to rewrite your queries for the new analyzer.
+
+### Invalid queries are no longer optimized
+
+The previous query planning infrastructure applied AST-level optimizations before the query validation step.
+Optimizations could rewrite the initial query so it becomes valid and can be executed.
+
+In the new analyzer, query validation takes place before the optimization step.
+This means that invalid queries that were possible to execute before are now unsupported.
+In such cases, the query must be fixed manually.
+
+**Example 1:**
+
+```sql
+SELECT number
+FROM numbers(1)
+GROUP BY toString(number)
+```
+
+The following query uses column `number` in the projection list when only `toString(number)` is available after the aggregation.
+In the old analyzer, `GROUP BY toString(number)` was optimized into `GROUP BY number,` making the query valid.
+
+**Example 2:**
+
+```sql
+SELECT
+ number % 2 AS n,
+ sum(number)
+FROM numbers(10)
+GROUP BY n
+HAVING number > 5
+```
+
+The same problem occurs in this query: column `number` is used after aggregation with another key.
+The previous query analyzer fixed this query by moving the `number > 5` filter from the `HAVING` clause to the `WHERE` clause.
+
+To fix the query, you should move all conditions that apply to non-aggregated columns to the `WHERE` section to conform to standard SQL syntax:
+```sql
+SELECT
+ number % 2 AS n,
+ sum(number)
+FROM numbers(10)
+WHERE number > 5
+GROUP BY n
+```
+
+### CREATE VIEW with invalid query
+
+The new analyzer always performs type-checking.
+Previously, it was possible to create a `VIEW` with an invalid `SELECT` query. It would then fail during the first `SELECT` or `INSERT` (in the case of `MATERIALIZED VIEW`).
+
+Now, it's not possible to create such `VIEW`s anymore.
+
+**Example:**
+
+```sql
+CREATE TABLE source (data String) ENGINE=MergeTree ORDER BY tuple();
+
+CREATE VIEW some_view
+AS SELECT JSONExtract(data, 'test', 'DateTime64(3)')
+FROM source;
+```
+
+### Known incompatibilities of the `JOIN` clause
+
+#### Join using column from projection
+
+Alias from the `SELECT` list can not be used as a `JOIN USING` key by default.
+
+A new setting, `analyzer_compatibility_join_using_top_level_identifier`, when enabled, alters the behavior of `JOIN USING` to prefer to resolve identifiers based on expressions from the projection list of the `SELECT` query, rather than using the columns from left table directly.
+
+**Example:**
+
+```sql
+SELECT a + 1 AS b, t2.s
+FROM Values('a UInt64, b UInt64', (1, 1)) AS t1
+JOIN Values('b UInt64, s String', (1, 'one'), (2, 'two')) t2
+USING (b);
+```
+
+With `analyzer_compatibility_join_using_top_level_identifier` set to `true`, the join condition is interpreted as `t1.a + 1 = t2.b`, matching the behavior of earlier versions. So, the result will be `2, 'two'`.
+When the setting is `false`, the join condition defaults to `t1.b = t2.b`, and the query will return `2, 'one'`.
+If `b` is not present in `t1`, the query will fail with an error.
+
+#### Changes in behavior with `JOIN USING` and `ALIAS`/`MATERIALIZED` columns
+
+In the new analyzer, using `*` in a `JOIN USING` query that involves `ALIAS` or `MATERIALIZED` columns will include those columns in the result set by default.
+
+**Example:**
+
+```sql
+CREATE TABLE t1 (id UInt64, payload ALIAS sipHash64(id)) ENGINE = MergeTree ORDER BY id;
+INSERT INTO t1 VALUES (1), (2);
+
+CREATE TABLE t2 (id UInt64, payload ALIAS sipHash64(id)) ENGINE = MergeTree ORDER BY id;
+INSERT INTO t2 VALUES (2), (3);
+
+SELECT * FROM t1
+FULL JOIN t2 USING (payload);
+```
+
+In the new analyzer, the result of this query will include the `payload` column along with `id` from both tables. In contrast, the previous analyzer would only include these `ALIAS` columns if specific settings (`asterisk_include_alias_columns` or `asterisk_include_materialized_columns`) were enabled, and the columns might appear in a different order.
+
+To ensure consistent and expected results, especially when migrating old queries to the new analyzer, it is advisable to specify columns explicitly in the `SELECT` clause rather than using `*`.
+
+#### Handling of Type Modifiers for columns in `USING` Clause
+
+In the new version of the analyzer, the rules for determining the common supertype for columns specified in the `USING` clause have been standardized to produce more predictable outcomes, especially when dealing with type modifiers like `LowCardinality` and `Nullable`.
+
+- `LowCardinality(T)` and `T`: When a column of type `LowCardinality(T)` is joined with a column of type `T`, the resulting common supertype will be `T`, effectively discarding the `LowCardinality` modifier.
+
+- `Nullable(T)` and `T`: When a column of type `Nullable(T)` is joined with a column of type `T`, the resulting common supertype will be `Nullable(T)`, ensuring that the nullable property is preserved.
+
+**Example:**
+
+```sql
+SELECT id, toTypeName(id) FROM Values('id LowCardinality(String)', ('a')) AS t1
+FULL OUTER JOIN Values('id String', ('b')) AS t2
+USING (id);
+```
+
+In this query, the common supertype for `id` is determined as `String`, discarding the `LowCardinality` modifier from `t1`.
+
+### Projection column names changes
+
+During projection names computation, aliases are not substituted.
+
+```sql
+SELECT
+ 1 + 1 AS x,
+ x + 1
+SETTINGS allow_experimental_analyzer = 0
+FORMAT PrettyCompact
+
+ ┌─x─┬─plus(plus(1, 1), 1)─┐
+1. │ 2 │ 3 │
+ └───┴─────────────────────┘
+
+SELECT
+ 1 + 1 AS x,
+ x + 1
+SETTINGS allow_experimental_analyzer = 1
+FORMAT PrettyCompact
+
+ ┌─x─┬─plus(x, 1)─┐
+1. │ 2 │ 3 │
+ └───┴────────────┘
+```
+
+### Incompatible function arguments types
+
+In the new analyzer, type inference happens during initial query analysis.
+This change means that type checks are done before short-circuit evaluation; thus, `if` function arguments must always have a common supertype.
+
+**Example:**
+
+The following query fails with `There is no supertype for types Array(UInt8), String because some of them are Array and some of them are not`:
+
+```sql
+SELECT toTypeName(if(0, [2, 3, 4], 'String'))
+```
+
+### Heterogeneous clusters
+
+The new analyzer significantly changed the communication protocol between servers in the cluster. Thus, it's impossible to run distributed queries on servers with different `allow_experimental_analyzer` setting values.
+
+### Mutations are interpreted by previous analyzer
+
+Mutations are still using the old analyzer.
+This means some new ClickHouse SQL features can't be used in mutations. For example, the `QUALIFY` clause.
+Status can be checked [here](https://github.com/ClickHouse/ClickHouse/issues/61563).
+
+### Unsupported features
+
+The list of features new analyzer currently doesn't support:
+
+- Annoy index.
+- Hypothesis index. Work in progress [here](https://github.com/ClickHouse/ClickHouse/pull/48381).
+- Window view is not supported. There are no plans to support it in the future.
diff --git a/docs/en/operations/external-authenticators/ssl-x509.md b/docs/en/operations/external-authenticators/ssl-x509.md
index 109913c2b18..09fac45d7ae 100644
--- a/docs/en/operations/external-authenticators/ssl-x509.md
+++ b/docs/en/operations/external-authenticators/ssl-x509.md
@@ -6,23 +6,30 @@ import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.m
-[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` field of the certificate is used to identify connected user. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration.
+[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` or `subjectAltName extension` field of the certificate is used to identify the connected user. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration.
-To enable SSL certificate authentication, a list of `Common Name`'s for each ClickHouse user must be specified in the settings file `users.xml `:
+To enable SSL certificate authentication, a list of `Common Name`'s or `Subject Alt Name`'s for each ClickHouse user must be specified in the settings file `users.xml `:
**Example**
```xml
-
+
host.domain.com:example_user
host.domain.com:example_user_dev
-
+
+
+
+ DNS:host.domain.com
+
+
+
+
```
diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md
index c9d94dd95ee..59ee05d1f9e 100644
--- a/docs/en/operations/named-collections.md
+++ b/docs/en/operations/named-collections.md
@@ -5,6 +5,10 @@ sidebar_label: "Named collections"
title: "Named collections"
---
+import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
+
+
+
Named collections provide a way to store collections of key-value pairs to be
used to configure integrations with external sources. You can use named collections with
dictionaries, tables, table functions, and object storage.
@@ -67,6 +71,23 @@ To manage named collections with DDL a user must have the `named_control_collect
In the above example the `password_sha256_hex` value is the hexadecimal representation of the SHA256 hash of the password. This configuration for the user `default` has the attribute `replace=true` as in the default configuration has a plain text `password` set, and it is not possible to have both plain text and sha256 hex passwords set for a user.
:::
+### Storage for named collections
+
+Named collections can either be stored on local disk or in zookeeper/keeper. By default local storage is used.
+
+To configure named collections storage in keeper and a `type` (equal to either `keeper` or `zookeeper`) and `path` (path in keeper, where named collections will be stored) to `named_collections_storage` section in configuration file:
+```
+
+
+ zookeeper
+ /named_collections_path/
+ 1000
+
+
+```
+
+An optional configuration parameter `update_timeout_ms` by default is equal to `5000`.
+
## Storing named collections in configuration files
### XML example
@@ -443,3 +464,59 @@ SELECT dictGet('dict', 'b', 1);
│ a │
└─────────────────────────┘
```
+
+## Named collections for accessing Kafka
+
+The description of parameters see [Kafka](../engines/table-engines/integrations/kafka.md).
+
+### DDL example
+
+```sql
+CREATE NAMED COLLECTION my_kafka_cluster AS
+kafka_broker_list = 'localhost:9092',
+kafka_topic_list = 'kafka_topic',
+kafka_group_name = 'consumer_group',
+kafka_format = 'JSONEachRow',
+kafka_max_block_size = '1048576';
+
+```
+### XML example
+
+```xml
+
+
+
+ localhost:9092
+ kafka_topic
+ consumer_group
+ JSONEachRow
+ 1048576
+
+
+
+```
+
+### Example of using named collections with a Kafka table
+
+Both of the following examples use the same named collection `my_kafka_cluster`:
+
+
+```sql
+CREATE TABLE queue
+(
+ timestamp UInt64,
+ level String,
+ message String
+)
+ENGINE = Kafka(my_kafka_cluster)
+
+CREATE TABLE queue
+(
+ timestamp UInt64,
+ level String,
+ message String
+)
+ENGINE = Kafka(my_kafka_cluster)
+SETTINGS kafka_num_consumers = 4,
+ kafka_thread_per_consumer = 1;
+```
diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md
index a5fe74fd0c6..8278f8c8699 100644
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@@ -498,6 +498,8 @@ Default: 0.9
Interval in seconds during which the server's maximum allowed memory consumption is adjusted by the corresponding threshold in cgroups. (see
settings `cgroup_memory_watcher_hard_limit_ratio` and `cgroup_memory_watcher_soft_limit_ratio`).
+To disable the cgroup observer, set this value to `0`.
+
Type: UInt64
Default: 15
@@ -591,6 +593,22 @@ Default value: 100000
400
```
+## max\_table\_num\_to\_throw {#max-table-num-to-throw}
+If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.Default value: 0
+
+**Example**
+```xml
+400
+```
+
+## max\_database\_num\_to\_throw {#max-table-num-to-throw}
+If number of _database is greater than this value, server will throw an exception. 0 means no limitation.
+Default value: 0
+
+**Example**
+```xml
+400
+```
## max_temporary_data_on_disk_size
@@ -938,6 +956,38 @@ Or it can be set in hex:
Everything mentioned above can be applied for `aes_256_gcm_siv` (but the key must be 32 bytes long).
+## error_log {#error_log}
+
+It is disabled by default.
+
+**Enabling**
+
+To manually turn on error history collection [`system.error_log`](../../operations/system-tables/error_log.md), create `/etc/clickhouse-server/config.d/error_log.xml` with the following content:
+
+``` xml
+
+
+ system
+
+ 7500
+ 1000
+ 1048576
+ 8192
+ 524288
+ false
+
+
+```
+
+**Disabling**
+
+To disable `error_log` setting, you should create the following file `/etc/clickhouse-server/config.d/disable_error_log.xml` with the following content:
+
+``` xml
+
+
+
+```
## custom_settings_prefixes {#custom_settings_prefixes}
@@ -1206,6 +1256,16 @@ Expired time for HSTS in seconds. The default value is 0 means clickhouse disabl
600000
```
+## mlock_executable {#mlock_executable}
+
+Perform mlockall after startup to lower first queries latency and to prevent clickhouse executable from being paged out under high IO load. Enabling this option is recommended but will lead to increased startup time for up to a few seconds.
+Keep in mind that this parameter would not work without "CAP_IPC_LOCK" capability.
+**Example**
+
+``` xml
+false
+```
+
## include_from {#include_from}
The path to the file with substitutions. Both XML and YAML formats are supported.
@@ -1353,6 +1413,26 @@ Examples:
127.0.0.1
```
+## listen_try {#listen_try}
+
+The server will not exit if IPv6 or IPv4 networks are unavailable while trying to listen.
+
+Examples:
+
+``` xml
+0
+```
+
+## listen_reuse_port {#listen_reuse_port}
+
+Allow multiple servers to listen on the same address:port. Requests will be routed to a random server by the operating system. Enabling this setting is not recommended.
+
+Examples:
+
+``` xml
+0
+```
+
## listen_backlog {#listen_backlog}
Backlog (queue size of pending connections) of the listen socket.
@@ -1385,6 +1465,9 @@ Keys:
- `size` – Size of the file. Applies to `log` and `errorlog`. Once the file reaches `size`, ClickHouse archives and renames it, and creates a new log file in its place.
- `count` – The number of archived log files that ClickHouse stores.
- `console` – Send `log` and `errorlog` to the console instead of file. To enable, set to `1` or `true`.
+- `console_log_level` – Logging level for console. Default to `level`.
+- `use_syslog` - Log to syslog as well.
+- `syslog_level` - Logging level for logging to syslog.
- `stream_compress` – Compress `log` and `errorlog` with `lz4` stream compression. To enable, set to `1` or `true`.
- `formatting` – Specify log format to be printed in console log (currently only `json` supported).
@@ -1871,7 +1954,7 @@ For more information, see the MergeTreeSettings.h header file.
## metric_log {#metric_log}
-It is enabled by default. If it`s not, you can do this manually.
+It is disabled by default.
**Enabling**
@@ -2894,6 +2977,8 @@ Define proxy servers for HTTP and HTTPS requests, currently supported by S3 stor
There are three ways to define proxy servers: environment variables, proxy lists, and remote proxy resolvers.
+Bypassing proxy servers for specific hosts is also supported with the use of `no_proxy`.
+
### Environment variables
The `http_proxy` and `https_proxy` environment variables allow you to specify a
@@ -3003,6 +3088,29 @@ This also allows a mix of resolver types can be used.
By default, tunneling (i.e, `HTTP CONNECT`) is used to make `HTTPS` requests over `HTTP` proxy. This setting can be used to disable it.
+### no_proxy
+By default, all requests will go through the proxy. In order to disable it for specific hosts, the `no_proxy` variable must be set.
+It can be set inside the `` clause for list and remote resolvers and as an environment variable for environment resolver.
+It supports IP addresses, domains, subdomains and `'*'` wildcard for full bypass. Leading dots are stripped just like curl does.
+
+Example:
+
+The below configuration bypasses proxy requests to `clickhouse.cloud` and all of its subdomains (e.g, `auth.clickhouse.cloud`).
+The same applies to GitLab, even though it has a leading dot. Both `gitlab.com` and `about.gitlab.com` would bypass the proxy.
+
+``` xml
+
+ clickhouse.cloud,.gitlab.com
+
+ http://proxy1
+ http://proxy2:3128
+
+
+ http://proxy1:3128
+
+
+```
+
## max_materialized_views_count_for_table {#max_materialized_views_count_for_table}
A limit on the number of materialized views attached to a table.
@@ -3029,3 +3137,21 @@ This setting is only necessary for the migration period and will become obsolete
Type: Bool
Default: 1
+
+## merge_workload {#merge_workload}
+
+Used to regulate how resources are utilized and shared between merges and other workloads. Specified value is used as `workload` setting value for all background merges. Can be overridden by a merge tree setting.
+
+Default value: "default"
+
+**See Also**
+- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
+
+## mutation_workload {#mutation_workload}
+
+Used to regulate how resources are utilized and shared between mutations and other workloads. Specified value is used as `workload` setting value for all background mutations. Can be overridden by a merge tree setting.
+
+Default value: "default"
+
+**See Also**
+- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md
index c3f303dcd38..9879ee35612 100644
--- a/docs/en/operations/settings/merge-tree-settings.md
+++ b/docs/en/operations/settings/merge-tree-settings.md
@@ -3,9 +3,126 @@ slug: /en/operations/settings/merge-tree-settings
title: "MergeTree tables settings"
---
-The values of `merge_tree` settings (for all MergeTree tables) can be viewed in the table `system.merge_tree_settings`, they can be overridden in `config.xml` in the `merge_tree` section, or set in the `SETTINGS` section of each table.
+System table `system.merge_tree_settings` shows the globally set MergeTree settings.
-These are example overrides for `max_suspicious_broken_parts`:
+MergeTree settings can be set in the `merge_tree` section of the server config file, or specified for each `MergeTree` table individually in
+the `SETTINGS` clause of the `CREATE TABLE` statement.
+
+Example for customizing setting `max_suspicious_broken_parts`:
+
+Configure the default for all `MergeTree` tables in the server configuration file:
+
+``` text
+
+ 5
+
+```
+
+Set for a particular table:
+
+``` sql
+CREATE TABLE tab
+(
+ `A` Int64
+)
+ENGINE = MergeTree
+ORDER BY tuple()
+SETTINGS max_suspicious_broken_parts = 500;
+```
+
+Change the settings for a particular table using `ALTER TABLE ... MODIFY SETTING`:
+
+```sql
+ALTER TABLE tab MODIFY SETTING max_suspicious_broken_parts = 100;
+
+-- reset to global default (value from system.merge_tree_settings)
+ALTER TABLE tab RESET SETTING max_suspicious_broken_parts;
+```
+
+## index_granularity
+
+Maximum number of data rows between the marks of an index.
+
+Default value: 8192.
+
+## index_granularity_bytes
+
+Maximum size of data granules in bytes.
+
+Default value: 10Mb.
+
+To restrict the granule size only by number of rows, set to 0 (not recommended).
+
+## min_index_granularity_bytes
+
+Min allowed size of data granules in bytes.
+
+Default value: 1024b.
+
+To provide a safeguard against accidentally creating tables with very low index_granularity_bytes.
+
+## enable_mixed_granularity_parts
+
+Enables or disables transitioning to control the granule size with the `index_granularity_bytes` setting. Before version 19.11, there was only the `index_granularity` setting for restricting granule size. The `index_granularity_bytes` setting improves ClickHouse performance when selecting data from tables with big rows (tens and hundreds of megabytes). If you have tables with big rows, you can enable this setting for the tables to improve the efficiency of `SELECT` queries.
+
+## use_minimalistic_part_header_in_zookeeper
+
+Storage method of the data parts headers in ZooKeeper. If enabled, ZooKeeper stores less data. For details, see [here](../server-configuration-parameters/settings.md/#server-settings-use_minimalistic_part_header_in_zookeeper).
+
+## min_merge_bytes_to_use_direct_io
+
+The minimum data volume for merge operation that is required for using direct I/O access to the storage disk.
+When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged.
+If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, ClickHouse reads and writes the data to the storage disk using the direct I/O interface (`O_DIRECT` option).
+If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled.
+
+Default value: `10 * 1024 * 1024 * 1024` bytes.
+
+## merge_with_ttl_timeout
+
+Minimum delay in seconds before repeating a merge with delete TTL.
+
+Default value: `14400` seconds (4 hours).
+
+## merge_with_recompression_ttl_timeout
+
+Minimum delay in seconds before repeating a merge with recompression TTL.
+
+Default value: `14400` seconds (4 hours).
+
+## write_final_mark
+
+Enables or disables writing the final index mark at the end of data part (after the last byte).
+
+Default value: 1.
+
+Don’t change or bad things will happen.
+
+## storage_policy
+
+Storage policy.
+
+## min_bytes_for_wide_part
+
+Minimum number of bytes/rows in a data part that can be stored in `Wide` format.
+You can set one, both or none of these settings.
+
+## max_compress_block_size
+
+Maximum size of blocks of uncompressed data before compressing for writing to a table.
+You can also specify this setting in the global settings (see [max_compress_block_size](/docs/en/operations/settings/settings.md/#max-compress-block-size) setting).
+The value specified when table is created overrides the global value for this setting.
+
+## min_compress_block_size
+
+Minimum size of blocks of uncompressed data required for compression when writing the next mark.
+You can also specify this setting in the global settings (see [min_compress_block_size](/docs/en/operations/settings/settings.md/#min-compress-block-size) setting).
+The value specified when table is created overrides the global value for this setting.
+
+## max_partitions_to_read
+
+Limits the maximum number of partitions that can be accessed in one query.
+You can also specify setting [max_partitions_to_read](/docs/en/operations/settings/merge-tree-settings.md/#max-partitions-to-read) in the global setting.
## max_suspicious_broken_parts
@@ -17,37 +134,6 @@ Possible values:
Default value: 100.
-Override example in `config.xml`:
-
-``` text
-
- 5
-
-```
-
-An example to set in `SETTINGS` for a particular table:
-
-``` sql
-CREATE TABLE foo
-(
- `A` Int64
-)
-ENGINE = MergeTree
-ORDER BY tuple()
-SETTINGS max_suspicious_broken_parts = 500;
-```
-
-An example of changing the settings for a specific table with the `ALTER TABLE ... MODIFY SETTING` command:
-
-``` sql
-ALTER TABLE foo
- MODIFY SETTING max_suspicious_broken_parts = 100;
-
--- reset to default (use value from system.merge_tree_settings)
-ALTER TABLE foo
- RESET SETTING max_suspicious_broken_parts;
-```
-
## parts_to_throw_insert {#parts-to-throw-insert}
If the number of active parts in a single partition exceeds the `parts_to_throw_insert` value, `INSERT` is interrupted with the `Too many parts (N). Merges are processing significantly slower than inserts` exception.
@@ -301,6 +387,8 @@ Default value: 10800
## try_fetch_recompressed_part_timeout
+Timeout (in seconds) before starting merge with recompression. During this time ClickHouse tries to fetch recompressed part from replica which assigned this merge with recompression.
+
Recompression works slow in most cases, so we don't start merge with recompression until this timeout and trying to fetch recompressed part from replica which assigned this merge with recompression.
Possible values:
@@ -886,10 +974,30 @@ Default value: false
- [exclude_deleted_rows_for_part_size_in_merge](#exclude_deleted_rows_for_part_size_in_merge) setting
-### allow_experimental_optimized_row_order
+## merge_workload
+
+Used to regulate how resources are utilized and shared between merges and other workloads. Specified value is used as `workload` setting value for background merges of this table. If not specified (empty string), then server setting `merge_workload` is used instead.
+
+Default value: an empty string
+
+**See Also**
+- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
+
+## mutation_workload
+
+Used to regulate how resources are utilized and shared between mutations and other workloads. Specified value is used as `workload` setting value for background mutations of this table. If not specified (empty string), then server setting `mutation_workload` is used instead.
+
+Default value: an empty string
+
+**See Also**
+- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
+
+### optimize_row_order
Controls if the row order should be optimized during inserts to improve the compressability of the newly inserted table part.
+Only has an effect for ordinary MergeTree-engine tables. Does nothing for specialized MergeTree engine tables (e.g. CollapsingMergeTree).
+
MergeTree tables are (optionally) compressed using [compression codecs](../../sql-reference/statements/create/table.md#column_compression_codec).
Generic compression codecs such as LZ4 and ZSTD achieve maximum compression rates if the data exposes patterns.
Long runs of the same value typically compress very well.
diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md
index 1a27b350652..530023df5b7 100644
--- a/docs/en/operations/settings/settings-formats.md
+++ b/docs/en/operations/settings/settings-formats.md
@@ -1417,6 +1417,24 @@ Compression method used in output Parquet format. Supported codecs: `snappy`, `l
Default value: `lz4`.
+### input_format_parquet_max_block_size {#input_format_parquet_max_block_size}
+Max block row size for parquet reader. By controlling the number of rows in each block, you can control the memory usage,
+and in some operators that cache blocks, you can improve the accuracy of the operator's memory control。
+
+Default value: `65409`.
+
+### input_format_parquet_prefer_block_bytes {#input_format_parquet_prefer_block_bytes}
+Average block bytes output by parquet reader. Lowering the configuration in the case of reading some high compression parquet relieves the memory pressure.
+
+Default value: `65409 * 256 = 16744704`
+
+### output_format_parquet_write_page_index {#input_format_parquet_max_block_size}
+
+Could add page index into parquet files. To enable this, need set `output_format_parquet_use_custom_encoder`=`false` and
+`output_format_parquet_write_page_index`=`true`.
+
+Enable by default.
+
## Hive format settings {#hive-format-settings}
### input_format_hive_text_fields_delimiter {#input_format_hive_text_fields_delimiter}
@@ -1695,6 +1713,43 @@ Result:
└────────────┘
```
+## output_format_pretty_display_footer_column_names
+
+Display column names in the footer if there are many table rows.
+
+Possible values:
+
+- 0 — No column names are displayed in the footer.
+- 1 — Column names are displayed in the footer if row count is greater than or equal to the threshold value set by [output_format_pretty_display_footer_column_names_min_rows](#output_format_pretty_display_footer_column_names_min_rows) (50 by default).
+
+Default value: `1`.
+
+**Example**
+
+Query:
+
+```sql
+SELECT *, toTypeName(*) FROM (SELECT * FROM system.numbers LIMIT 1000);
+```
+
+Result:
+
+```response
+ ┌─number─┬─toTypeName(number)─┐
+ 1. │ 0 │ UInt64 │
+ 2. │ 1 │ UInt64 │
+ 3. │ 2 │ UInt64 │
+ ...
+ 999. │ 998 │ UInt64 │
+1000. │ 999 │ UInt64 │
+ └─number─┴─toTypeName(number)─┘
+```
+## output_format_pretty_display_footer_column_names_min_rows
+
+Sets the minimum number of rows for which a footer with column names will be displayed if setting [output_format_pretty_display_footer_column_names](#output_format_pretty_display_footer_column_names) is enabled.
+
+Default value: `50`.
+
## Template format settings {#template-format-settings}
### format_template_resultset {#format_template_resultset}
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 0b905df21d4..3d6d776f4da 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1590,6 +1590,22 @@ Possible values:
Default value: `default`.
+## parallel_replicas_custom_key_range_lower {#parallel_replicas_custom_key_range_lower}
+
+Allows the filter type `range` to split the work evenly between replicas based on the custom range `[parallel_replicas_custom_key_range_lower, INT_MAX]`.
+
+When used in conjuction with [parallel_replicas_custom_key_range_upper](#parallel_replicas_custom_key_range_upper), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
+
+Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
+
+## parallel_replicas_custom_key_range_upper {#parallel_replicas_custom_key_range_upper}
+
+Allows the filter type `range` to split the work evenly between replicas based on the custom range `[0, parallel_replicas_custom_key_range_upper]`. A value of 0 disables the upper bound, setting it the max value of the custom key expression.
+
+When used in conjuction with [parallel_replicas_custom_key_range_lower](#parallel_replicas_custom_key_range_lower), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
+
+Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
+
## allow_experimental_parallel_reading_from_replicas
Enables or disables sending SELECT queries to all replicas of a table (up to `max_parallel_replicas`). Reading is parallelized and coordinated dynamically. It will work for any kind of MergeTree table.
@@ -3170,6 +3186,18 @@ Possible values:
Default value: `0`.
+## lightweight_deletes_sync {#lightweight_deletes_sync}
+
+The same as 'mutation_sync', but controls only execution of lightweight deletes.
+
+Possible values:
+
+- 0 - Mutations execute asynchronously.
+- 1 - The query waits for the lightweight deletes to complete on the current server.
+- 2 - The query waits for the lightweight deletes to complete on all replicas (if they exist).
+
+Default value: `2`.
+
**See Also**
- [Synchronicity of ALTER Queries](../../sql-reference/statements/alter/index.md#synchronicity-of-alter-queries)
@@ -3850,6 +3878,10 @@ Possible values:
Default value: 30.
+:::note
+It's applicable only to the default profile. A server reboot is required for the changes to take effect.
+:::
+
## http_receive_timeout {#http_receive_timeout}
HTTP receive timeout (in seconds).
@@ -5108,7 +5140,7 @@ a Tuple(
)
```
-## allow_experimental_statistic {#allow_experimental_statistic}
+## allow_experimental_statistics {#allow_experimental_statistics}
Allows defining columns with [statistics](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) and [manipulate statistics](../../engines/table-engines/mergetree-family/mergetree.md#column-statistics).
@@ -5118,7 +5150,7 @@ Allows using statistic to optimize the order of [prewhere conditions](../../sql-
## analyze_index_with_space_filling_curves
-If a table has a space-filling curve in its index, e.g. `ORDER BY mortonEncode(x, y)`, and the query has conditions on its arguments, e.g. `x >= 10 AND x <= 20 AND y >= 20 AND y <= 30`, use the space-filling curve for index analysis.
+If a table has a space-filling curve in its index, e.g. `ORDER BY mortonEncode(x, y)` or `ORDER BY hilbertEncode(x, y)`, and the query has conditions on its arguments, e.g. `x >= 10 AND x <= 20 AND y >= 20 AND y <= 30`, use the space-filling curve for index analysis.
## query_plan_enable_optimizations {#query_plan_enable_optimizations}
@@ -5386,6 +5418,15 @@ When set to `false` than all attempts are made with identical timeouts.
Default value: `true`.
+## allow_deprecated_snowflake_conversion_functions {#allow_deprecated_snowflake_conversion_functions}
+
+Functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake` are deprecated and disabled by default.
+Please use functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` instead.
+
+To re-enable the deprecated functions (e.g., during a transition period), please set this setting to `true`.
+
+Default value: `false`
+
## allow_experimental_variant_type {#allow_experimental_variant_type}
Allows creation of experimental [Variant](../../sql-reference/data-types/variant.md).
diff --git a/docs/en/operations/system-tables/asynchronous_metrics.md b/docs/en/operations/system-tables/asynchronous_metrics.md
index 81725b97e41..762d187917c 100644
--- a/docs/en/operations/system-tables/asynchronous_metrics.md
+++ b/docs/en/operations/system-tables/asynchronous_metrics.md
@@ -639,6 +639,10 @@ An internal metric of the low-level memory allocator (jemalloc). See https://jem
An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html
+### jemalloc.prof.active
+
+An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html
+
**See Also**
- [Monitoring](../../operations/monitoring.md) — Base concepts of ClickHouse monitoring.
diff --git a/docs/en/operations/system-tables/error_log.md b/docs/en/operations/system-tables/error_log.md
new file mode 100644
index 00000000000..15edef58662
--- /dev/null
+++ b/docs/en/operations/system-tables/error_log.md
@@ -0,0 +1,39 @@
+---
+slug: /en/operations/system-tables/error_log
+---
+# error_log
+
+Contains history of error values from table `system.errors`, periodically flushed to disk.
+
+Columns:
+- `hostname` ([LowCardinality(String)](../../sql-reference/data-types/string.md)) — Hostname of the server executing the query.
+- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
+- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
+- `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — Code number of the error.
+- `error` ([LowCardinality(String)](../../sql-reference/data-types/string.md)) - Name of the error.
+- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of times this error happened.
+- `remote` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Remote exception (i.e. received during one of the distributed queries).
+
+**Example**
+
+``` sql
+SELECT * FROM system.error_log LIMIT 1 FORMAT Vertical;
+```
+
+``` text
+Row 1:
+──────
+hostname: clickhouse.eu-central1.internal
+event_date: 2024-06-18
+event_time: 2024-06-18 07:32:39
+code: 999
+error: KEEPER_EXCEPTION
+value: 2
+remote: 0
+```
+
+**See also**
+
+- [error_log setting](../../operations/server-configuration-parameters/settings.md#error_log) — Enabling and disabling the setting.
+- [system.errors](../../operations/system-tables/errors.md) — Contains error codes with the number of times they have been triggered.
+- [Monitoring](../../operations/monitoring.md) — Base concepts of ClickHouse monitoring.
diff --git a/docs/en/operations/system-tables/query_log.md b/docs/en/operations/system-tables/query_log.md
index 75b855966a3..47094eec3f0 100644
--- a/docs/en/operations/system-tables/query_log.md
+++ b/docs/en/operations/system-tables/query_log.md
@@ -113,6 +113,8 @@ Columns:
- `used_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `functions`, which were used during query execution.
- `used_storages` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `storages`, which were used during query execution.
- `used_table_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `table functions`, which were used during query execution.
+- `used_privileges` ([Array(String)](../../sql-reference/data-types/array.md)) - Privileges which were successfully checked during query execution.
+- `missing_privileges` ([Array(String)](../../sql-reference/data-types/array.md)) - Privileges that are missing during query execution.
- `query_cache_usage` ([Enum8](../../sql-reference/data-types/enum.md)) — Usage of the [query cache](../query-cache.md) during query execution. Values:
- `'Unknown'` = Status unknown.
- `'None'` = The query result was neither written into nor read from the query cache.
@@ -194,6 +196,8 @@ used_formats: []
used_functions: []
used_storages: []
used_table_functions: []
+used_privileges: []
+missing_privileges: []
query_cache_usage: None
```
diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md
index ed22679a3e6..df041f5885e 100644
--- a/docs/en/operations/tips.md
+++ b/docs/en/operations/tips.md
@@ -36,9 +36,24 @@ $ echo 0 | sudo tee /proc/sys/vm/overcommit_memory
Use `perf top` to watch the time spent in the kernel for memory management.
Permanent huge pages also do not need to be allocated.
-:::warning
-If your system has less than 16 GB of RAM, you may experience various memory exceptions because default settings do not match this amount of memory. The recommended amount of RAM is 32 GB or more. You can use ClickHouse in a system with a small amount of RAM, even with 2 GB of RAM, but it requires additional tuning and can ingest at a low rate.
-:::
+### Using less than 16GB of RAM
+
+The recommended amount of RAM is 32 GB or more.
+
+If your system has less than 16 GB of RAM, you may experience various memory exceptions because default settings do not match this amount of memory. You can use ClickHouse in a system with a small amount of RAM (as low as 2 GB), but these setups require additional tuning and can only ingest at a low rate.
+
+When using ClickHouse with less than 16GB of RAM, we recommend the following:
+
+- Lower the size of the mark cache in the `config.xml`. It can be set as low as 500 MB, but it cannot be set to zero.
+- Lower the number of query processing threads down to `1`.
+- Lower the `max_block_size` to `8192`. Values as low as `1024` can still be practical.
+- Lower `max_download_threads` to `1`.
+- Set `input_format_parallel_parsing` and `output_format_parallel_formatting` to `0`.
+
+Additional notes:
+- To flush the memory cached by the memory allocator, you can run the `SYSTEM JEMALLOC PURGE`
+command.
+- We do not recommend using S3 or Kafka integrations on low-memory machines because they require significant memory for buffers.
## Storage Subsystem {#storage-subsystem}
diff --git a/docs/en/operations/utilities/clickhouse-local.md b/docs/en/operations/utilities/clickhouse-local.md
index 93a3fecf3c6..f19643a3fa5 100644
--- a/docs/en/operations/utilities/clickhouse-local.md
+++ b/docs/en/operations/utilities/clickhouse-local.md
@@ -236,10 +236,10 @@ Read 2 rows, 32.00 B in 0.000 sec., 5182 rows/sec., 80.97 KiB/sec.
Previous example is the same as:
``` bash
-$ echo -e "1,2\n3,4" | clickhouse-local --query "
+$ echo -e "1,2\n3,4" | clickhouse-local -n --query "
CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin);
SELECT a, b FROM table;
- DROP TABLE table"
+ DROP TABLE table;"
Read 2 rows, 32.00 B in 0.000 sec., 4987 rows/sec., 77.93 KiB/sec.
1 2
3 4
diff --git a/docs/en/operations/utilities/odbc-bridge.md b/docs/en/operations/utilities/odbc-bridge.md
index abb8860880e..eb849c6b6ae 100644
--- a/docs/en/operations/utilities/odbc-bridge.md
+++ b/docs/en/operations/utilities/odbc-bridge.md
@@ -18,7 +18,7 @@ This tool works via HTTP, not via pipes, shared memory, or TCP because:
However it can be used as standalone tool from command line with the following
parameters in POST-request URL:
- `connection_string` -- ODBC connection string.
-- `columns` -- columns in ClickHouse NamesAndTypesList format, name in backticks,
+- `sample_block` -- columns description in ClickHouse NamesAndTypesList format, name in backticks,
type as string. Name and type are space separated, rows separated with
newline.
- `max_block_size` -- optional parameter, sets maximum size of single block.
diff --git a/docs/en/operations/workload-scheduling.md b/docs/en/operations/workload-scheduling.md
index 24149099892..08629492ec6 100644
--- a/docs/en/operations/workload-scheduling.md
+++ b/docs/en/operations/workload-scheduling.md
@@ -47,6 +47,8 @@ Example:
Queries can be marked with setting `workload` to distinguish different workloads. If `workload` is not set, than value "default" is used. Note that you are able to specify the other value using settings profiles. Setting constraints can be used to make `workload` constant if you want all queries from the user to be marked with fixed value of `workload` setting.
+It is possible to assign a `workload` setting for background activities. Merges and mutations are using `merge_workload` and `mutation_workload` server settings correspondingly. These values can also be overridden for specific tables using `merge_workload` and `mutation_workload` merge tree settings
+
Let's consider an example of a system with two different workloads: "production" and "development".
```sql
@@ -151,6 +153,9 @@ Example:
```
-
## See also
- [system.scheduler](/docs/en/operations/system-tables/scheduler.md)
+ - [merge_workload](/docs/en/operations/settings/merge-tree-settings.md#merge_workload) merge tree setting
+ - [merge_workload](/docs/en/operations/server-configuration-parameters/settings.md#merge_workload) global server setting
+ - [mutation_workload](/docs/en/operations/settings/merge-tree-settings.md#mutation_workload) merge tree setting
+ - [mutation_workload](/docs/en/operations/server-configuration-parameters/settings.md#mutation_workload) global server setting
diff --git a/docs/en/sql-reference/aggregate-functions/combinators.md b/docs/en/sql-reference/aggregate-functions/combinators.md
index 8ccc5e292b5..e30aa66b3b3 100644
--- a/docs/en/sql-reference/aggregate-functions/combinators.md
+++ b/docs/en/sql-reference/aggregate-functions/combinators.md
@@ -106,8 +106,8 @@ To work with these states, use:
- [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) table engine.
- [finalizeAggregation](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) function.
- [runningAccumulate](../../sql-reference/functions/other-functions.md#runningaccumulate) function.
-- [-Merge](#aggregate_functions_combinators-merge) combinator.
-- [-MergeState](#aggregate_functions_combinators-mergestate) combinator.
+- [-Merge](#-merge) combinator.
+- [-MergeState](#-mergestate) combinator.
## -Merge
diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md
index 1dc89b8dcf9..093d88f939f 100644
--- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md
+++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md
@@ -82,10 +82,12 @@ FROM
In this case, you should remember that you do not know the histogram bin borders.
-## sequenceMatch(pattern)(timestamp, cond1, cond2, ...)
+## sequenceMatch
Checks whether the sequence contains an event chain that matches the pattern.
+**Syntax**
+
``` sql
sequenceMatch(pattern)(timestamp, cond1, cond2, ...)
```
@@ -102,7 +104,7 @@ Events that occur at the same second may lay in the sequence in an undefined ord
**Parameters**
-- `pattern` — Pattern string. See [Pattern syntax](#sequence-function-pattern-syntax).
+- `pattern` — Pattern string. See [Pattern syntax](#sequencematch).
**Returned values**
@@ -170,9 +172,9 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM
**See Also**
-- [sequenceCount](#function-sequencecount)
+- [sequenceCount](#sequencecount)
-## sequenceCount(pattern)(time, cond1, cond2, ...)
+## sequenceCount
Counts the number of event chains that matched the pattern. The function searches event chains that do not overlap. It starts to search for the next chain after the current chain is matched.
@@ -180,6 +182,8 @@ Counts the number of event chains that matched the pattern. The function searche
Events that occur at the same second may lay in the sequence in an undefined order affecting the result.
:::
+**Syntax**
+
``` sql
sequenceCount(pattern)(timestamp, cond1, cond2, ...)
```
@@ -192,7 +196,7 @@ sequenceCount(pattern)(timestamp, cond1, cond2, ...)
**Parameters**
-- `pattern` — Pattern string. See [Pattern syntax](#sequence-function-pattern-syntax).
+- `pattern` — Pattern string. See [Pattern syntax](#sequencematch).
**Returned values**
@@ -229,7 +233,7 @@ SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t
**See Also**
-- [sequenceMatch](#function-sequencematch)
+- [sequenceMatch](#sequencematch)
## windowFunnel
diff --git a/docs/en/sql-reference/aggregate-functions/reference/analysis_of_variance.md b/docs/en/sql-reference/aggregate-functions/reference/analysis_of_variance.md
index d9b44b3ff07..56e54d3faf9 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/analysis_of_variance.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/analysis_of_variance.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/analysis_of_variance
-sidebar_position: 6
+sidebar_position: 101
---
# analysisOfVariance
diff --git a/docs/en/sql-reference/aggregate-functions/reference/any.md b/docs/en/sql-reference/aggregate-functions/reference/any.md
index f1b5a6683e5..cdff7dde4a9 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/any.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/any.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/any
-sidebar_position: 6
+sidebar_position: 102
---
# any
diff --git a/docs/en/sql-reference/aggregate-functions/reference/anyheavy.md b/docs/en/sql-reference/aggregate-functions/reference/anyheavy.md
index 9fbc21910f8..9c6e6b5fead 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/anyheavy.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/anyheavy.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/anyheavy
-sidebar_position: 103
+sidebar_position: 104
---
# anyHeavy
diff --git a/docs/en/sql-reference/aggregate-functions/reference/anylast.md b/docs/en/sql-reference/aggregate-functions/reference/anylast.md
index 8fcee2cf8e6..e43bc07fbdc 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/anylast.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/anylast.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/anylast
-sidebar_position: 104
+sidebar_position: 105
---
# anyLast
diff --git a/docs/en/sql-reference/aggregate-functions/reference/anylast_respect_nulls.md b/docs/en/sql-reference/aggregate-functions/reference/anylast_respect_nulls.md
index b6d0806f35d..8f093cfdb61 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/anylast_respect_nulls.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/anylast_respect_nulls.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/anylast_respect_nulls
-sidebar_position: 104
+sidebar_position: 106
---
# anyLast_respect_nulls
diff --git a/docs/en/sql-reference/aggregate-functions/reference/approxtopk.md b/docs/en/sql-reference/aggregate-functions/reference/approxtopk.md
index 2bb43a9f665..ea2083ebd04 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/approxtopk.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/approxtopk.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/approxtopk
-sidebar_position: 212
+sidebar_position: 107
---
# approx_top_k
diff --git a/docs/en/sql-reference/aggregate-functions/reference/approxtopsum.md b/docs/en/sql-reference/aggregate-functions/reference/approxtopsum.md
index aa884b26d8e..639142331f0 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/approxtopsum.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/approxtopsum.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/approxtopsum
-sidebar_position: 212
+sidebar_position: 108
---
# approx_top_sum
diff --git a/docs/en/sql-reference/aggregate-functions/reference/argmax.md b/docs/en/sql-reference/aggregate-functions/reference/argmax.md
index 2274dd4a5dc..8c6b2b532e8 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/argmax.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/argmax.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/argmax
-sidebar_position: 106
+sidebar_position: 109
---
# argMax
diff --git a/docs/en/sql-reference/aggregate-functions/reference/argmin.md b/docs/en/sql-reference/aggregate-functions/reference/argmin.md
index 297744fb1db..0ab21fe2b52 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/argmin.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/argmin.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/argmin
-sidebar_position: 105
+sidebar_position: 110
---
# argMin
diff --git a/docs/en/sql-reference/aggregate-functions/reference/arrayconcatagg.md b/docs/en/sql-reference/aggregate-functions/reference/arrayconcatagg.md
index 3c71129bdb5..c0ac0db33f3 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/arrayconcatagg.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/arrayconcatagg.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/array_concat_agg
-sidebar_position: 110
+sidebar_position: 111
---
# array_concat_agg
diff --git a/docs/en/sql-reference/aggregate-functions/reference/avg.md b/docs/en/sql-reference/aggregate-functions/reference/avg.md
index 5463d8a1874..7789c30bfe0 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/avg.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/avg.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/avg
-sidebar_position: 5
+sidebar_position: 112
---
# avg
diff --git a/docs/en/sql-reference/aggregate-functions/reference/avgweighted.md b/docs/en/sql-reference/aggregate-functions/reference/avgweighted.md
index 99d3bac763d..304d0407d98 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/avgweighted.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/avgweighted.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/avgweighted
-sidebar_position: 107
+sidebar_position: 113
---
# avgWeighted
diff --git a/docs/en/sql-reference/aggregate-functions/reference/boundrat.md b/docs/en/sql-reference/aggregate-functions/reference/boundrat.md
index f3907af8030..d253a250600 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/boundrat.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/boundrat.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/boundingRatio
-sidebar_position: 2
+sidebar_position: 114
title: boundingRatio
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/categoricalinformationvalue.md b/docs/en/sql-reference/aggregate-functions/reference/categoricalinformationvalue.md
index 57edb47950a..7983c3f2e60 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/categoricalinformationvalue.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/categoricalinformationvalue.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/categoricalinformationvalue
-sidebar_position: 250
+sidebar_position: 115
title: categoricalInformationValue
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/contingency.md b/docs/en/sql-reference/aggregate-functions/reference/contingency.md
index 902c1f4af80..a49ff22febc 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/contingency.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/contingency.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/contingency
-sidebar_position: 350
+sidebar_position: 116
---
# contingency
diff --git a/docs/en/sql-reference/aggregate-functions/reference/corr.md b/docs/en/sql-reference/aggregate-functions/reference/corr.md
index 5681c942169..c43b4d3b25a 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/corr.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/corr.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/corr
-sidebar_position: 107
+sidebar_position: 117
---
# corr
diff --git a/docs/en/sql-reference/aggregate-functions/reference/corrmatrix.md b/docs/en/sql-reference/aggregate-functions/reference/corrmatrix.md
index 718477b28dd..96978863646 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/corrmatrix.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/corrmatrix.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/corrmatrix
-sidebar_position: 108
+sidebar_position: 118
---
# corrMatrix
diff --git a/docs/en/sql-reference/aggregate-functions/reference/corrstable.md b/docs/en/sql-reference/aggregate-functions/reference/corrstable.md
index b35442a32b6..979cf244245 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/corrstable.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/corrstable.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/corrstable
-sidebar_position: 107
+sidebar_position: 119
---
# corrStable
diff --git a/docs/en/sql-reference/aggregate-functions/reference/count.md b/docs/en/sql-reference/aggregate-functions/reference/count.md
index ca4067c8d8c..e6f2cdd6aa9 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/count.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/count.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/count
-sidebar_position: 1
+sidebar_position: 120
---
# count
diff --git a/docs/en/sql-reference/aggregate-functions/reference/covarpop.md b/docs/en/sql-reference/aggregate-functions/reference/covarpop.md
index 78b9f4cffea..7231f92b8fa 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/covarpop.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/covarpop.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/covarpop
-sidebar_position: 37
+sidebar_position: 121
---
# covarPop
diff --git a/docs/en/sql-reference/aggregate-functions/reference/covarpopmatrix.md b/docs/en/sql-reference/aggregate-functions/reference/covarpopmatrix.md
index d7400599a49..c8811b3811e 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/covarpopmatrix.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/covarpopmatrix.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/covarpopmatrix
-sidebar_position: 36
+sidebar_position: 122
---
# covarPopMatrix
diff --git a/docs/en/sql-reference/aggregate-functions/reference/covarpopstable.md b/docs/en/sql-reference/aggregate-functions/reference/covarpopstable.md
index 68e78fc3bd8..48e5368faac 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/covarpopstable.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/covarpopstable.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/covarpopstable
-sidebar_position: 36
+sidebar_position: 123
---
# covarPopStable
diff --git a/docs/en/sql-reference/aggregate-functions/reference/covarsamp.md b/docs/en/sql-reference/aggregate-functions/reference/covarsamp.md
index 7d5d5d13f35..92fe213b407 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/covarsamp.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/covarsamp.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/covarsamp
-sidebar_position: 37
+sidebar_position: 124
---
# covarSamp
diff --git a/docs/en/sql-reference/aggregate-functions/reference/covarsampmatrix.md b/docs/en/sql-reference/aggregate-functions/reference/covarsampmatrix.md
index b71d753f0be..1585c4a9970 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/covarsampmatrix.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/covarsampmatrix.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/covarsampmatrix
-sidebar_position: 38
+sidebar_position: 125
---
# covarSampMatrix
diff --git a/docs/en/sql-reference/aggregate-functions/reference/covarsampstable.md b/docs/en/sql-reference/aggregate-functions/reference/covarsampstable.md
index 3e6867b96d6..6764877768e 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/covarsampstable.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/covarsampstable.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/covarsampstable
-sidebar_position: 37
+sidebar_position: 126
---
# covarSampStable
diff --git a/docs/en/sql-reference/aggregate-functions/reference/cramersv.md b/docs/en/sql-reference/aggregate-functions/reference/cramersv.md
index 2424ff95237..db0e1c5eb4c 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/cramersv.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/cramersv.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/cramersv
-sidebar_position: 351
+sidebar_position: 127
---
# cramersV
diff --git a/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md b/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md
index 939c04e3fdc..2ff7ce489d3 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/cramersvbiascorrected.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/cramersvbiascorrected
-sidebar_position: 352
+sidebar_position: 128
---
# cramersVBiasCorrected
diff --git a/docs/en/sql-reference/aggregate-functions/reference/deltasum.md b/docs/en/sql-reference/aggregate-functions/reference/deltasum.md
index 37d9d08cbdb..650135ecfeb 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/deltasum.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/deltasum.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/deltasum
-sidebar_position: 141
+sidebar_position: 129
---
# deltaSum
diff --git a/docs/en/sql-reference/aggregate-functions/reference/deltasumtimestamp.md b/docs/en/sql-reference/aggregate-functions/reference/deltasumtimestamp.md
index c51d86389b0..ec5cfa5fecc 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/deltasumtimestamp.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/deltasumtimestamp.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/deltasumtimestamp
-sidebar_position: 141
+sidebar_position: 130
title: deltaSumTimestamp
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/entropy.md b/docs/en/sql-reference/aggregate-functions/reference/entropy.md
index fc8d627ecab..7970cdd268b 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/entropy.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/entropy.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/entropy
-sidebar_position: 302
+sidebar_position: 131
---
# entropy
diff --git a/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md b/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md
index 75041ace7a3..3086a48f819 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md
@@ -1,7 +1,7 @@
---
-slug: /en/sql-reference/aggregate-functions/reference/exponentialmovingaverage
-sidebar_position: 108
-sidebar_title: exponentialMovingAverage
+slug: /en/sql-reference/aggregate-functions/reference/exponentialMovingAverage
+sidebar_position: 132
+title: exponentialMovingAverage
---
## exponentialMovingAverage
@@ -96,56 +96,56 @@ Result:
``` text
┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────────────────────────────────────┐
-│ 1 │ 0 │ 0.067 │ ███▎ │
+│ 1 │ 0 │ 0.067 │ ███▎ │
│ 0 │ 1 │ 0.062 │ ███ │
-│ 0 │ 2 │ 0.058 │ ██▊ │
-│ 0 │ 3 │ 0.054 │ ██▋ │
+│ 0 │ 2 │ 0.058 │ ██▊ │
+│ 0 │ 3 │ 0.054 │ ██▋ │
│ 0 │ 4 │ 0.051 │ ██▌ │
-│ 0 │ 5 │ 0.047 │ ██▎ │
-│ 0 │ 6 │ 0.044 │ ██▏ │
+│ 0 │ 5 │ 0.047 │ ██▎ │
+│ 0 │ 6 │ 0.044 │ ██▏ │
│ 0 │ 7 │ 0.041 │ ██ │
-│ 0 │ 8 │ 0.038 │ █▊ │
-│ 0 │ 9 │ 0.036 │ █▋ │
-│ 0 │ 10 │ 0.033 │ █▋ │
+│ 0 │ 8 │ 0.038 │ █▊ │
+│ 0 │ 9 │ 0.036 │ █▋ │
+│ 0 │ 10 │ 0.033 │ █▋ │
│ 0 │ 11 │ 0.031 │ █▌ │
-│ 0 │ 12 │ 0.029 │ █▍ │
-│ 0 │ 13 │ 0.027 │ █▎ │
-│ 0 │ 14 │ 0.025 │ █▎ │
-│ 0 │ 15 │ 0.024 │ █▏ │
+│ 0 │ 12 │ 0.029 │ █▍ │
+│ 0 │ 13 │ 0.027 │ █▎ │
+│ 0 │ 14 │ 0.025 │ █▎ │
+│ 0 │ 15 │ 0.024 │ █▏ │
│ 0 │ 16 │ 0.022 │ █ │
│ 0 │ 17 │ 0.021 │ █ │
-│ 0 │ 18 │ 0.019 │ ▊ │
-│ 0 │ 19 │ 0.018 │ ▊ │
-│ 0 │ 20 │ 0.017 │ ▋ │
-│ 0 │ 21 │ 0.016 │ ▋ │
-│ 0 │ 22 │ 0.015 │ ▋ │
-│ 0 │ 23 │ 0.014 │ ▋ │
-│ 0 │ 24 │ 0.013 │ ▋ │
-│ 1 │ 25 │ 0.079 │ ███▊ │
+│ 0 │ 18 │ 0.019 │ ▊ │
+│ 0 │ 19 │ 0.018 │ ▊ │
+│ 0 │ 20 │ 0.017 │ ▋ │
+│ 0 │ 21 │ 0.016 │ ▋ │
+│ 0 │ 22 │ 0.015 │ ▋ │
+│ 0 │ 23 │ 0.014 │ ▋ │
+│ 0 │ 24 │ 0.013 │ ▋ │
+│ 1 │ 25 │ 0.079 │ ███▊ │
│ 1 │ 26 │ 0.14 │ ███████ │
-│ 1 │ 27 │ 0.198 │ █████████▊ │
+│ 1 │ 27 │ 0.198 │ █████████▊ │
│ 1 │ 28 │ 0.252 │ ████████████▌ │
│ 1 │ 29 │ 0.302 │ ███████████████ │
-│ 1 │ 30 │ 0.349 │ █████████████████▍ │
+│ 1 │ 30 │ 0.349 │ █████████████████▍ │
│ 1 │ 31 │ 0.392 │ ███████████████████▌ │
-│ 1 │ 32 │ 0.433 │ █████████████████████▋ │
+│ 1 │ 32 │ 0.433 │ █████████████████████▋ │
│ 1 │ 33 │ 0.471 │ ███████████████████████▌ │
-│ 1 │ 34 │ 0.506 │ █████████████████████████▎ │
-│ 1 │ 35 │ 0.539 │ ██████████████████████████▊ │
+│ 1 │ 34 │ 0.506 │ █████████████████████████▎ │
+│ 1 │ 35 │ 0.539 │ ██████████████████████████▊ │
│ 1 │ 36 │ 0.57 │ ████████████████████████████▌ │
-│ 1 │ 37 │ 0.599 │ █████████████████████████████▊ │
-│ 1 │ 38 │ 0.626 │ ███████████████████████████████▎ │
+│ 1 │ 37 │ 0.599 │ █████████████████████████████▊ │
+│ 1 │ 38 │ 0.626 │ ███████████████████████████████▎ │
│ 1 │ 39 │ 0.651 │ ████████████████████████████████▌ │
-│ 1 │ 40 │ 0.674 │ █████████████████████████████████▋ │
-│ 1 │ 41 │ 0.696 │ ██████████████████████████████████▋ │
-│ 1 │ 42 │ 0.716 │ ███████████████████████████████████▋ │
-│ 1 │ 43 │ 0.735 │ ████████████████████████████████████▋ │
-│ 1 │ 44 │ 0.753 │ █████████████████████████████████████▋ │
-│ 1 │ 45 │ 0.77 │ ██████████████████████████████████████▍ │
-│ 1 │ 46 │ 0.785 │ ███████████████████████████████████████▎ │
-│ 1 │ 47 │ 0.8 │ ███████████████████████████████████████▊ │
-│ 1 │ 48 │ 0.813 │ ████████████████████████████████████████▋ │
-│ 1 │ 49 │ 0.825 │ █████████████████████████████████████████▎│
+│ 1 │ 40 │ 0.674 │ █████████████████████████████████▋ │
+│ 1 │ 41 │ 0.696 │ ██████████████████████████████████▋ │
+│ 1 │ 42 │ 0.716 │ ███████████████████████████████████▋ │
+│ 1 │ 43 │ 0.735 │ ████████████████████████████████████▋ │
+│ 1 │ 44 │ 0.753 │ █████████████████████████████████████▋ │
+│ 1 │ 45 │ 0.77 │ ██████████████████████████████████████▍ │
+│ 1 │ 46 │ 0.785 │ ███████████████████████████████████████▎ │
+│ 1 │ 47 │ 0.8 │ ███████████████████████████████████████▊ │
+│ 1 │ 48 │ 0.813 │ ████████████████████████████████████████▋ │
+│ 1 │ 49 │ 0.825 │ █████████████████████████████████████████▎ │
└───────┴──────┴──────────────────────┴────────────────────────────────────────────┘
```
diff --git a/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedavg.md b/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedavg.md
new file mode 100644
index 00000000000..c729552749a
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedavg.md
@@ -0,0 +1,105 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/exponentialTimeDecayedAvg
+sidebar_position: 133
+title: exponentialTimeDecayedAvg
+---
+
+## exponentialTimeDecayedAvg
+
+Returns the exponentially smoothed weighted moving average of values of a time series at point `t` in time.
+
+**Syntax**
+
+```sql
+exponentialTimeDecayedAvg(x)(v, t)
+```
+
+**Arguments**
+
+- `v` — Value. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
+- `t` — Time. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md), [DateTime](../../data-types/datetime.md), [DateTime64](../../data-types/datetime64.md).
+
+**Parameters**
+
+- `x` — Half-life period. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
+
+**Returned values**
+
+- Returns an exponentially smoothed weighted moving average at index `t` in time. [Float64](../../data-types/float.md).
+
+**Examples**
+
+Query:
+
+```sql
+SELECT
+ value,
+ time,
+ round(exp_smooth, 3),
+ bar(exp_smooth, 0, 5, 50) AS bar
+FROM
+ (
+ SELECT
+ (number = 0) OR (number >= 25) AS value,
+ number AS time,
+ exponentialTimeDecayedAvg(10)(value, time) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
+ FROM numbers(50)
+ );
+```
+
+Response:
+
+```sql
+ ┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────┐
+1. │ 1 │ 0 │ 1 │ ██████████ │
+2. │ 0 │ 1 │ 0.475 │ ████▊ │
+3. │ 0 │ 2 │ 0.301 │ ███ │
+4. │ 0 │ 3 │ 0.214 │ ██▏ │
+5. │ 0 │ 4 │ 0.162 │ █▌ │
+6. │ 0 │ 5 │ 0.128 │ █▎ │
+7. │ 0 │ 6 │ 0.104 │ █ │
+8. │ 0 │ 7 │ 0.086 │ ▊ │
+9. │ 0 │ 8 │ 0.072 │ ▋ │
+0. │ 0 │ 9 │ 0.061 │ ▌ │
+1. │ 0 │ 10 │ 0.052 │ ▌ │
+2. │ 0 │ 11 │ 0.045 │ ▍ │
+3. │ 0 │ 12 │ 0.039 │ ▍ │
+4. │ 0 │ 13 │ 0.034 │ ▎ │
+5. │ 0 │ 14 │ 0.03 │ ▎ │
+6. │ 0 │ 15 │ 0.027 │ ▎ │
+7. │ 0 │ 16 │ 0.024 │ ▏ │
+8. │ 0 │ 17 │ 0.021 │ ▏ │
+9. │ 0 │ 18 │ 0.018 │ ▏ │
+0. │ 0 │ 19 │ 0.016 │ ▏ │
+1. │ 0 │ 20 │ 0.015 │ ▏ │
+2. │ 0 │ 21 │ 0.013 │ ▏ │
+3. │ 0 │ 22 │ 0.012 │ │
+4. │ 0 │ 23 │ 0.01 │ │
+5. │ 0 │ 24 │ 0.009 │ │
+6. │ 1 │ 25 │ 0.111 │ █ │
+7. │ 1 │ 26 │ 0.202 │ ██ │
+8. │ 1 │ 27 │ 0.283 │ ██▊ │
+9. │ 1 │ 28 │ 0.355 │ ███▌ │
+0. │ 1 │ 29 │ 0.42 │ ████▏ │
+1. │ 1 │ 30 │ 0.477 │ ████▊ │
+2. │ 1 │ 31 │ 0.529 │ █████▎ │
+3. │ 1 │ 32 │ 0.576 │ █████▊ │
+4. │ 1 │ 33 │ 0.618 │ ██████▏ │
+5. │ 1 │ 34 │ 0.655 │ ██████▌ │
+6. │ 1 │ 35 │ 0.689 │ ██████▉ │
+7. │ 1 │ 36 │ 0.719 │ ███████▏ │
+8. │ 1 │ 37 │ 0.747 │ ███████▍ │
+9. │ 1 │ 38 │ 0.771 │ ███████▋ │
+0. │ 1 │ 39 │ 0.793 │ ███████▉ │
+1. │ 1 │ 40 │ 0.813 │ ████████▏ │
+2. │ 1 │ 41 │ 0.831 │ ████████▎ │
+3. │ 1 │ 42 │ 0.848 │ ████████▍ │
+4. │ 1 │ 43 │ 0.862 │ ████████▌ │
+5. │ 1 │ 44 │ 0.876 │ ████████▊ │
+6. │ 1 │ 45 │ 0.888 │ ████████▉ │
+7. │ 1 │ 46 │ 0.898 │ ████████▉ │
+8. │ 1 │ 47 │ 0.908 │ █████████ │
+9. │ 1 │ 48 │ 0.917 │ █████████▏ │
+0. │ 1 │ 49 │ 0.925 │ █████████▏ │
+ └───────┴──────┴──────────────────────┴────────────┘
+```
\ No newline at end of file
diff --git a/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedcount.md b/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedcount.md
new file mode 100644
index 00000000000..b73d6c2503d
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedcount.md
@@ -0,0 +1,104 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/exponentialTimeDecayedCount
+sidebar_position: 134
+title: exponentialTimeDecayedCount
+---
+
+## exponentialTimeDecayedCount
+
+Returns the cumulative exponential decay over a time series at the index `t` in time.
+
+**Syntax**
+
+```sql
+exponentialTimeDecayedCount(x)(t)
+```
+
+**Arguments**
+
+- `t` — Time. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md), [DateTime](../../data-types/datetime.md), [DateTime64](../../data-types/datetime64.md).
+
+**Parameters**
+
+- `x` — Half-life period. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
+
+**Returned values**
+
+- Returns the cumulative exponential decay at the given point in time. [Float64](../../data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+ value,
+ time,
+ round(exp_smooth, 3),
+ bar(exp_smooth, 0, 20, 50) AS bar
+FROM
+(
+ SELECT
+ (number % 5) = 0 AS value,
+ number AS time,
+ exponentialTimeDecayedCount(10)(time) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
+ FROM numbers(50)
+);
+```
+
+Result:
+
+```response
+ ┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────────────────────┐
+ 1. │ 1 │ 0 │ 1 │ ██▌ │
+ 2. │ 0 │ 1 │ 1.905 │ ████▊ │
+ 3. │ 0 │ 2 │ 2.724 │ ██████▊ │
+ 4. │ 0 │ 3 │ 3.464 │ ████████▋ │
+ 5. │ 0 │ 4 │ 4.135 │ ██████████▎ │
+ 6. │ 1 │ 5 │ 4.741 │ ███████████▊ │
+ 7. │ 0 │ 6 │ 5.29 │ █████████████▏ │
+ 8. │ 0 │ 7 │ 5.787 │ ██████████████▍ │
+ 9. │ 0 │ 8 │ 6.236 │ ███████████████▌ │
+10. │ 0 │ 9 │ 6.643 │ ████████████████▌ │
+11. │ 1 │ 10 │ 7.01 │ █████████████████▌ │
+12. │ 0 │ 11 │ 7.343 │ ██████████████████▎ │
+13. │ 0 │ 12 │ 7.644 │ ███████████████████ │
+14. │ 0 │ 13 │ 7.917 │ ███████████████████▊ │
+15. │ 0 │ 14 │ 8.164 │ ████████████████████▍ │
+16. │ 1 │ 15 │ 8.387 │ ████████████████████▉ │
+17. │ 0 │ 16 │ 8.589 │ █████████████████████▍ │
+18. │ 0 │ 17 │ 8.771 │ █████████████████████▉ │
+19. │ 0 │ 18 │ 8.937 │ ██████████████████████▎ │
+20. │ 0 │ 19 │ 9.086 │ ██████████████████████▋ │
+21. │ 1 │ 20 │ 9.222 │ ███████████████████████ │
+22. │ 0 │ 21 │ 9.344 │ ███████████████████████▎ │
+23. │ 0 │ 22 │ 9.455 │ ███████████████████████▋ │
+24. │ 0 │ 23 │ 9.555 │ ███████████████████████▉ │
+25. │ 0 │ 24 │ 9.646 │ ████████████████████████ │
+26. │ 1 │ 25 │ 9.728 │ ████████████████████████▎ │
+27. │ 0 │ 26 │ 9.802 │ ████████████████████████▌ │
+28. │ 0 │ 27 │ 9.869 │ ████████████████████████▋ │
+29. │ 0 │ 28 │ 9.93 │ ████████████████████████▊ │
+30. │ 0 │ 29 │ 9.985 │ ████████████████████████▉ │
+31. │ 1 │ 30 │ 10.035 │ █████████████████████████ │
+32. │ 0 │ 31 │ 10.08 │ █████████████████████████▏ │
+33. │ 0 │ 32 │ 10.121 │ █████████████████████████▎ │
+34. │ 0 │ 33 │ 10.158 │ █████████████████████████▍ │
+35. │ 0 │ 34 │ 10.191 │ █████████████████████████▍ │
+36. │ 1 │ 35 │ 10.221 │ █████████████████████████▌ │
+37. │ 0 │ 36 │ 10.249 │ █████████████████████████▌ │
+38. │ 0 │ 37 │ 10.273 │ █████████████████████████▋ │
+39. │ 0 │ 38 │ 10.296 │ █████████████████████████▋ │
+40. │ 0 │ 39 │ 10.316 │ █████████████████████████▊ │
+41. │ 1 │ 40 │ 10.334 │ █████████████████████████▊ │
+42. │ 0 │ 41 │ 10.351 │ █████████████████████████▉ │
+43. │ 0 │ 42 │ 10.366 │ █████████████████████████▉ │
+44. │ 0 │ 43 │ 10.379 │ █████████████████████████▉ │
+45. │ 0 │ 44 │ 10.392 │ █████████████████████████▉ │
+46. │ 1 │ 45 │ 10.403 │ ██████████████████████████ │
+47. │ 0 │ 46 │ 10.413 │ ██████████████████████████ │
+48. │ 0 │ 47 │ 10.422 │ ██████████████████████████ │
+49. │ 0 │ 48 │ 10.43 │ ██████████████████████████ │
+50. │ 0 │ 49 │ 10.438 │ ██████████████████████████ │
+ └───────┴──────┴──────────────────────┴────────────────────────────┘
+```
diff --git a/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedmax.md b/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedmax.md
new file mode 100644
index 00000000000..06dc5313904
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedmax.md
@@ -0,0 +1,105 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/exponentialTimeDecayedMax
+sidebar_position: 135
+title: exponentialTimeDecayedMax
+---
+
+## exponentialTimeDecayedMax
+
+Returns the maximum of the computed exponentially smoothed moving average at index `t` in time with that at `t-1`.
+
+**Syntax**
+
+```sql
+exponentialTimeDecayedMax(x)(value, timeunit)
+```
+
+**Arguments**
+
+- `value` — Value. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
+- `timeunit` — Timeunit. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md), [DateTime](../../data-types/datetime.md), [DateTime64](../../data-types/datetime64.md).
+
+**Parameters**
+
+- `x` — Half-life period. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
+
+**Returned values**
+
+- Returns the maximum of the exponentially smoothed weighted moving average at `t` and `t-1`. [Float64](../../data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+ value,
+ time,
+ round(exp_smooth, 3),
+ bar(exp_smooth, 0, 5, 50) AS bar
+FROM
+ (
+ SELECT
+ (number = 0) OR (number >= 25) AS value,
+ number AS time,
+ exponentialTimeDecayedMax(10)(value, time) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
+ FROM numbers(50)
+ );
+```
+
+Result:
+
+```response
+ ┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar────────┐
+ 1. │ 1 │ 0 │ 1 │ ██████████ │
+ 2. │ 0 │ 1 │ 0.905 │ █████████ │
+ 3. │ 0 │ 2 │ 0.819 │ ████████▏ │
+ 4. │ 0 │ 3 │ 0.741 │ ███████▍ │
+ 5. │ 0 │ 4 │ 0.67 │ ██████▋ │
+ 6. │ 0 │ 5 │ 0.607 │ ██████ │
+ 7. │ 0 │ 6 │ 0.549 │ █████▍ │
+ 8. │ 0 │ 7 │ 0.497 │ ████▉ │
+ 9. │ 0 │ 8 │ 0.449 │ ████▍ │
+10. │ 0 │ 9 │ 0.407 │ ████ │
+11. │ 0 │ 10 │ 0.368 │ ███▋ │
+12. │ 0 │ 11 │ 0.333 │ ███▎ │
+13. │ 0 │ 12 │ 0.301 │ ███ │
+14. │ 0 │ 13 │ 0.273 │ ██▋ │
+15. │ 0 │ 14 │ 0.247 │ ██▍ │
+16. │ 0 │ 15 │ 0.223 │ ██▏ │
+17. │ 0 │ 16 │ 0.202 │ ██ │
+18. │ 0 │ 17 │ 0.183 │ █▊ │
+19. │ 0 │ 18 │ 0.165 │ █▋ │
+20. │ 0 │ 19 │ 0.15 │ █▍ │
+21. │ 0 │ 20 │ 0.135 │ █▎ │
+22. │ 0 │ 21 │ 0.122 │ █▏ │
+23. │ 0 │ 22 │ 0.111 │ █ │
+24. │ 0 │ 23 │ 0.1 │ █ │
+25. │ 0 │ 24 │ 0.091 │ ▉ │
+26. │ 1 │ 25 │ 1 │ ██████████ │
+27. │ 1 │ 26 │ 1 │ ██████████ │
+28. │ 1 │ 27 │ 1 │ ██████████ │
+29. │ 1 │ 28 │ 1 │ ██████████ │
+30. │ 1 │ 29 │ 1 │ ██████████ │
+31. │ 1 │ 30 │ 1 │ ██████████ │
+32. │ 1 │ 31 │ 1 │ ██████████ │
+33. │ 1 │ 32 │ 1 │ ██████████ │
+34. │ 1 │ 33 │ 1 │ ██████████ │
+35. │ 1 │ 34 │ 1 │ ██████████ │
+36. │ 1 │ 35 │ 1 │ ██████████ │
+37. │ 1 │ 36 │ 1 │ ██████████ │
+38. │ 1 │ 37 │ 1 │ ██████████ │
+39. │ 1 │ 38 │ 1 │ ██████████ │
+40. │ 1 │ 39 │ 1 │ ██████████ │
+41. │ 1 │ 40 │ 1 │ ██████████ │
+42. │ 1 │ 41 │ 1 │ ██████████ │
+43. │ 1 │ 42 │ 1 │ ██████████ │
+44. │ 1 │ 43 │ 1 │ ██████████ │
+45. │ 1 │ 44 │ 1 │ ██████████ │
+46. │ 1 │ 45 │ 1 │ ██████████ │
+47. │ 1 │ 46 │ 1 │ ██████████ │
+48. │ 1 │ 47 │ 1 │ ██████████ │
+49. │ 1 │ 48 │ 1 │ ██████████ │
+50. │ 1 │ 49 │ 1 │ ██████████ │
+ └───────┴──────┴──────────────────────┴────────────┘
+```
\ No newline at end of file
diff --git a/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedsum.md b/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedsum.md
new file mode 100644
index 00000000000..617cd265dac
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/exponentialtimedecayedsum.md
@@ -0,0 +1,105 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/exponentialTimeDecayedSum
+sidebar_position: 136
+title: exponentialTimeDecayedSum
+---
+
+## exponentialTimeDecayedSum
+
+Returns the sum of exponentially smoothed moving average values of a time series at the index `t` in time.
+
+**Syntax**
+
+```sql
+exponentialTimeDecayedSum(x)(v, t)
+```
+
+**Arguments**
+
+- `v` — Value. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
+- `t` — Time. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md), [DateTime](../../data-types/datetime.md), [DateTime64](../../data-types/datetime64.md).
+
+**Parameters**
+
+- `x` — Half-life period. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md).
+
+**Returned values**
+
+- Returns the sum of exponentially smoothed moving average values at the given point in time. [Float64](../../data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+ value,
+ time,
+ round(exp_smooth, 3),
+ bar(exp_smooth, 0, 10, 50) AS bar
+FROM
+ (
+ SELECT
+ (number = 0) OR (number >= 25) AS value,
+ number AS time,
+ exponentialTimeDecayedSum(10)(value, time) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS exp_smooth
+ FROM numbers(50)
+ );
+```
+
+Result:
+
+```response
+ ┌─value─┬─time─┬─round(exp_smooth, 3)─┬─bar───────────────────────────────────────────────┐
+ 1. │ 1 │ 0 │ 1 │ █████ │
+ 2. │ 0 │ 1 │ 0.905 │ ████▌ │
+ 3. │ 0 │ 2 │ 0.819 │ ████ │
+ 4. │ 0 │ 3 │ 0.741 │ ███▋ │
+ 5. │ 0 │ 4 │ 0.67 │ ███▎ │
+ 6. │ 0 │ 5 │ 0.607 │ ███ │
+ 7. │ 0 │ 6 │ 0.549 │ ██▋ │
+ 8. │ 0 │ 7 │ 0.497 │ ██▍ │
+ 9. │ 0 │ 8 │ 0.449 │ ██▏ │
+10. │ 0 │ 9 │ 0.407 │ ██ │
+11. │ 0 │ 10 │ 0.368 │ █▊ │
+12. │ 0 │ 11 │ 0.333 │ █▋ │
+13. │ 0 │ 12 │ 0.301 │ █▌ │
+14. │ 0 │ 13 │ 0.273 │ █▎ │
+15. │ 0 │ 14 │ 0.247 │ █▏ │
+16. │ 0 │ 15 │ 0.223 │ █ │
+17. │ 0 │ 16 │ 0.202 │ █ │
+18. │ 0 │ 17 │ 0.183 │ ▉ │
+19. │ 0 │ 18 │ 0.165 │ ▊ │
+20. │ 0 │ 19 │ 0.15 │ ▋ │
+21. │ 0 │ 20 │ 0.135 │ ▋ │
+22. │ 0 │ 21 │ 0.122 │ ▌ │
+23. │ 0 │ 22 │ 0.111 │ ▌ │
+24. │ 0 │ 23 │ 0.1 │ ▌ │
+25. │ 0 │ 24 │ 0.091 │ ▍ │
+26. │ 1 │ 25 │ 1.082 │ █████▍ │
+27. │ 1 │ 26 │ 1.979 │ █████████▉ │
+28. │ 1 │ 27 │ 2.791 │ █████████████▉ │
+29. │ 1 │ 28 │ 3.525 │ █████████████████▋ │
+30. │ 1 │ 29 │ 4.19 │ ████████████████████▉ │
+31. │ 1 │ 30 │ 4.791 │ ███████████████████████▉ │
+32. │ 1 │ 31 │ 5.335 │ ██████████████████████████▋ │
+33. │ 1 │ 32 │ 5.827 │ █████████████████████████████▏ │
+34. │ 1 │ 33 │ 6.273 │ ███████████████████████████████▎ │
+35. │ 1 │ 34 │ 6.676 │ █████████████████████████████████▍ │
+36. │ 1 │ 35 │ 7.041 │ ███████████████████████████████████▏ │
+37. │ 1 │ 36 │ 7.371 │ ████████████████████████████████████▊ │
+38. │ 1 │ 37 │ 7.669 │ ██████████████████████████████████████▎ │
+39. │ 1 │ 38 │ 7.939 │ ███████████████████████████████████████▋ │
+40. │ 1 │ 39 │ 8.184 │ ████████████████████████████████████████▉ │
+41. │ 1 │ 40 │ 8.405 │ ██████████████████████████████████████████ │
+42. │ 1 │ 41 │ 8.605 │ ███████████████████████████████████████████ │
+43. │ 1 │ 42 │ 8.786 │ ███████████████████████████████████████████▉ │
+44. │ 1 │ 43 │ 8.95 │ ████████████████████████████████████████████▊ │
+45. │ 1 │ 44 │ 9.098 │ █████████████████████████████████████████████▍ │
+46. │ 1 │ 45 │ 9.233 │ ██████████████████████████████████████████████▏ │
+47. │ 1 │ 46 │ 9.354 │ ██████████████████████████████████████████████▊ │
+48. │ 1 │ 47 │ 9.464 │ ███████████████████████████████████████████████▎ │
+49. │ 1 │ 48 │ 9.563 │ ███████████████████████████████████████████████▊ │
+50. │ 1 │ 49 │ 9.653 │ ████████████████████████████████████████████████▎ │
+ └───────┴──────┴──────────────────────┴───────────────────────────────────────────────────┘
+```
\ No newline at end of file
diff --git a/docs/en/sql-reference/aggregate-functions/reference/first_value.md b/docs/en/sql-reference/aggregate-functions/reference/first_value.md
index 0c26b66c64a..2cd0e1fa16f 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/first_value.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/first_value.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/first_value
-sidebar_position: 7
+sidebar_position: 137
---
# first_value
diff --git a/docs/en/sql-reference/aggregate-functions/reference/flame_graph.md b/docs/en/sql-reference/aggregate-functions/reference/flame_graph.md
new file mode 100644
index 00000000000..4abb3e03226
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/flame_graph.md
@@ -0,0 +1,95 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/flame_graph
+sidebar_position: 138
+---
+
+# flameGraph
+
+Aggregate function which builds a [flamegraph](https://www.brendangregg.com/flamegraphs.html) using the list of stacktraces. Outputs an array of strings which can be used by [flamegraph.pl utility](https://github.com/brendangregg/FlameGraph) to render an SVG of the flamegraph.
+
+## Syntax
+
+```sql
+flameGraph(traces, [size], [ptr])
+```
+
+## Parameters
+
+- `traces` — a stacktrace. [Array](../../data-types/array.md)([UInt64](../../data-types/int-uint.md)).
+- `size` — an allocation size for memory profiling. (optional - default `1`). [UInt64](../../data-types/int-uint.md).
+- `ptr` — an allocation address. (optional - default `0`). [UInt64](../../data-types/int-uint.md).
+
+:::note
+In the case where `ptr != 0`, a flameGraph will map allocations (size > 0) and deallocations (size < 0) with the same size and ptr.
+Only allocations which were not freed are shown. Non mapped deallocations are ignored.
+:::
+
+## Returned value
+
+- An array of strings for use with [flamegraph.pl utility](https://github.com/brendangregg/FlameGraph). [Array](../../data-types/array.md)([String](../../data-types/string.md)).
+
+## Examples
+
+### Building a flamegraph based on a CPU query profiler
+
+```sql
+SET query_profiler_cpu_time_period_ns=10000000;
+SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
+```
+
+```text
+clickhouse client --allow_introspection_functions=1 -q "select arrayJoin(flameGraph(arrayReverse(trace))) from system.trace_log where trace_type = 'CPU' and query_id = 'xxx'" | ~/dev/FlameGraph/flamegraph.pl > flame_cpu.svg
+```
+
+### Building a flamegraph based on a memory query profiler, showing all allocations
+
+```sql
+SET memory_profiler_sample_probability=1, max_untracked_memory=1;
+SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
+```
+
+```text
+clickhouse client --allow_introspection_functions=1 -q "select arrayJoin(flameGraph(trace, size)) from system.trace_log where trace_type = 'MemorySample' and query_id = 'xxx'" | ~/dev/FlameGraph/flamegraph.pl --countname=bytes --color=mem > flame_mem.svg
+```
+
+### Building a flamegraph based on a memory query profiler, showing allocations which were not deallocated in query context
+
+```sql
+SET memory_profiler_sample_probability=1, max_untracked_memory=1, use_uncompressed_cache=1, merge_tree_max_rows_to_use_cache=100000000000, merge_tree_max_bytes_to_use_cache=1000000000000;
+SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
+```
+
+```text
+clickhouse client --allow_introspection_functions=1 -q "SELECT arrayJoin(flameGraph(trace, size, ptr)) FROM system.trace_log WHERE trace_type = 'MemorySample' AND query_id = 'xxx'" | ~/dev/FlameGraph/flamegraph.pl --countname=bytes --color=mem > flame_mem_untracked.svg
+```
+
+### Build a flamegraph based on memory query profiler, showing active allocations at the fixed point of time
+
+```sql
+SET memory_profiler_sample_probability=1, max_untracked_memory=1;
+SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
+```
+
+- 1 - Memory usage per second
+
+```sql
+SELECT event_time, m, formatReadableSize(max(s) as m) FROM (SELECT event_time, sum(size) OVER (ORDER BY event_time) AS s FROM system.trace_log WHERE query_id = 'xxx' AND trace_type = 'MemorySample') GROUP BY event_time ORDER BY event_time;
+```
+
+- 2 - Find a time point with maximal memory usage
+
+```sql
+SELECT argMax(event_time, s), max(s) FROM (SELECT event_time, sum(size) OVER (ORDER BY event_time) AS s FROM system.trace_log WHERE query_id = 'xxx' AND trace_type = 'MemorySample');
+```
+
+- 3 - Fix active allocations at fixed point of time
+
+```text
+clickhouse client --allow_introspection_functions=1 -q "SELECT arrayJoin(flameGraph(trace, size, ptr)) FROM (SELECT * FROM system.trace_log WHERE trace_type = 'MemorySample' AND query_id = 'xxx' AND event_time <= 'yyy' ORDER BY event_time)" | ~/dev/FlameGraph/flamegraph.pl --countname=bytes --color=mem > flame_mem_time_point_pos.svg
+```
+
+- 4 - Find deallocations at fixed point of time
+
+```text
+clickhouse client --allow_introspection_functions=1 -q "SELECT arrayJoin(flameGraph(trace, -size, ptr)) FROM (SELECT * FROM system.trace_log WHERE trace_type = 'MemorySample' AND query_id = 'xxx' AND event_time > 'yyy' ORDER BY event_time desc)" | ~/dev/FlameGraph/flamegraph.pl --countname=bytes --color=mem > flame_mem_time_point_neg.svg
+```
diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparray.md b/docs/en/sql-reference/aggregate-functions/reference/grouparray.md
index a38e35a72ad..1a87e3aeba9 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/grouparray.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/grouparray.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/grouparray
-sidebar_position: 110
+sidebar_position: 139
---
# groupArray
diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat.md b/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat.md
index d745e8a0e7a..c6b23c2f808 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/grouparrayinsertat
-sidebar_position: 112
+sidebar_position: 140
---
# groupArrayInsertAt
diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparrayintersect.md b/docs/en/sql-reference/aggregate-functions/reference/grouparrayintersect.md
index 5cac88be073..a370f595923 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/grouparrayintersect.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/grouparrayintersect.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/grouparrayintersect
-sidebar_position: 115
+sidebar_position: 141
---
# groupArrayIntersect
diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparraylast.md b/docs/en/sql-reference/aggregate-functions/reference/grouparraylast.md
index 9b48ee54ecd..ff62dcdde9b 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/grouparraylast.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/grouparraylast.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/grouparraylast
-sidebar_position: 110
+sidebar_position: 142
---
# groupArrayLast
diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingavg.md b/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingavg.md
index 32c0608afeb..6b6c4830535 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingavg.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingavg.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/grouparraymovingavg
-sidebar_position: 114
+sidebar_position: 143
---
# groupArrayMovingAvg
diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingsum.md b/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingsum.md
index 6f2a60dd080..d1fa6fce9b0 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingsum.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingsum.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/grouparraymovingsum
-sidebar_position: 113
+sidebar_position: 144
---
# groupArrayMovingSum
diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparraysample.md b/docs/en/sql-reference/aggregate-functions/reference/grouparraysample.md
index 393087161df..38ddae48ee7 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/grouparraysample.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/grouparraysample.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/grouparraysample
-sidebar_position: 114
+sidebar_position: 145
---
# groupArraySample
diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparraysorted.md b/docs/en/sql-reference/aggregate-functions/reference/grouparraysorted.md
index 9bee0c29e7a..22a150bb8fb 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/grouparraysorted.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/grouparraysorted.md
@@ -1,6 +1,7 @@
- ---
- toc_priority: 112
- ---
+---
+slug: /en/sql-reference/aggregate-functions/reference/grouparraysorted
+sidebar_position: 146
+---
# groupArraySorted {#groupArraySorted}
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md
index 3d833555a43..eee383d84e9 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/groupbitand
-sidebar_position: 125
+sidebar_position: 147
---
# groupBitAnd
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitmap.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitmap.md
index 02b9e0e8821..23b686e29b2 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitmap.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitmap.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/groupbitmap
-sidebar_position: 128
+sidebar_position: 148
---
# groupBitmap
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitmapand.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitmapand.md
index 1e649645e75..77bbf7d3d2c 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitmapand.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitmapand.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/groupbitmapand
-sidebar_position: 129
+sidebar_position: 149
title: groupBitmapAnd
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitmapor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitmapor.md
index c88c80ceff2..7bb3dc689e8 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitmapor.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitmapor.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/groupbitmapor
-sidebar_position: 130
+sidebar_position: 150
title: groupBitmapOr
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitmapxor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitmapxor.md
index aa24b3d2128..3212e94a47b 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitmapxor.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitmapxor.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/groupbitmapxor
-sidebar_position: 131
+sidebar_position: 151
title: groupBitmapXor
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md
index 138ee998405..802b839d56e 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/groupbitor
-sidebar_position: 126
+sidebar_position: 152
---
# groupBitOr
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md
index 168335a010c..94891891d64 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/groupbitxor
-sidebar_position: 127
+sidebar_position: 153
---
# groupBitXor
diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupuniqarray.md b/docs/en/sql-reference/aggregate-functions/reference/groupuniqarray.md
index fe5f714c307..0462f4a4ab2 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupuniqarray.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupuniqarray.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/groupuniqarray
-sidebar_position: 111
+sidebar_position: 154
---
# groupUniqArray
diff --git a/docs/en/sql-reference/aggregate-functions/reference/index.md b/docs/en/sql-reference/aggregate-functions/reference/index.md
index a56b1c97681..e3725b6a430 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/index.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/index.md
@@ -58,6 +58,7 @@ ClickHouse-specific aggregate functions:
- [topKWeighted](../reference/topkweighted.md)
- [deltaSum](../reference/deltasum.md)
- [deltaSumTimestamp](../reference/deltasumtimestamp.md)
+- [flameGraph](../reference/flame_graph.md)
- [groupArray](../reference/grouparray.md)
- [groupArrayLast](../reference/grouparraylast.md)
- [groupUniqArray](../reference/groupuniqarray.md)
diff --git a/docs/en/sql-reference/aggregate-functions/reference/intervalLengthSum.md b/docs/en/sql-reference/aggregate-functions/reference/intervalLengthSum.md
index 5990345b765..66e23a716ba 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/intervalLengthSum.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/intervalLengthSum.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/intervalLengthSum
-sidebar_position: 146
+sidebar_position: 155
sidebar_label: intervalLengthSum
title: intervalLengthSum
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md b/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md
index d159eec7ce6..33afcdfbf38 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest
-sidebar_position: 300
+sidebar_position: 156
sidebar_label: kolmogorovSmirnovTest
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/kurtpop.md b/docs/en/sql-reference/aggregate-functions/reference/kurtpop.md
index e1a29973fcf..c543831addc 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/kurtpop.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/kurtpop.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/kurtpop
-sidebar_position: 153
+sidebar_position: 157
---
# kurtPop
diff --git a/docs/en/sql-reference/aggregate-functions/reference/kurtsamp.md b/docs/en/sql-reference/aggregate-functions/reference/kurtsamp.md
index 911c2bfbe74..57e80729454 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/kurtsamp.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/kurtsamp.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/kurtsamp
-sidebar_position: 154
+sidebar_position: 158
---
# kurtSamp
diff --git a/docs/en/sql-reference/aggregate-functions/reference/largestTriangleThreeBuckets.md b/docs/en/sql-reference/aggregate-functions/reference/largestTriangleThreeBuckets.md
index 06443994dd9..673f3cb69c7 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/largestTriangleThreeBuckets.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/largestTriangleThreeBuckets.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/largestTriangleThreeBuckets
-sidebar_position: 312
+sidebar_position: 159
sidebar_label: largestTriangleThreeBuckets
---
@@ -24,6 +24,8 @@ Alias: `lttb`.
- `x` — x coordinate. [Integer](../../../sql-reference/data-types/int-uint.md) , [Float](../../../sql-reference/data-types/float.md) , [Decimal](../../../sql-reference/data-types/decimal.md) , [Date](../../../sql-reference/data-types/date.md), [Date32](../../../sql-reference/data-types/date32.md), [DateTime](../../../sql-reference/data-types/datetime.md), [DateTime64](../../../sql-reference/data-types/datetime64.md).
- `y` — y coordinate. [Integer](../../../sql-reference/data-types/int-uint.md) , [Float](../../../sql-reference/data-types/float.md) , [Decimal](../../../sql-reference/data-types/decimal.md) , [Date](../../../sql-reference/data-types/date.md), [Date32](../../../sql-reference/data-types/date32.md), [DateTime](../../../sql-reference/data-types/datetime.md), [DateTime64](../../../sql-reference/data-types/datetime64.md).
+NaNs are ignored in the provided series, meaning that any NaN values will be excluded from the analysis. This ensures that the function operates only on valid numerical data.
+
**Parameters**
- `n` — number of points in the resulting series. [UInt64](../../../sql-reference/data-types/int-uint.md).
@@ -61,7 +63,7 @@ Result:
``` text
┌────────largestTriangleThreeBuckets(4)(x, y)───────────┐
-│ [(1,10),(3,15),(5,40),(10,70)] │
+│ [(1,10),(3,15),(9,55),(10,70)] │
└───────────────────────────────────────────────────────┘
```
diff --git a/docs/en/sql-reference/aggregate-functions/reference/last_value.md b/docs/en/sql-reference/aggregate-functions/reference/last_value.md
index 21a86a5f130..b2aa5c86d81 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/last_value.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/last_value.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/last_value
-sidebar_position: 8
+sidebar_position: 160
---
# last_value
diff --git a/docs/en/sql-reference/aggregate-functions/reference/mannwhitneyutest.md b/docs/en/sql-reference/aggregate-functions/reference/mannwhitneyutest.md
index af744f445d9..17f6afecde2 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/mannwhitneyutest.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/mannwhitneyutest.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/mannwhitneyutest
-sidebar_position: 310
+sidebar_position: 161
sidebar_label: mannWhitneyUTest
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/max.md b/docs/en/sql-reference/aggregate-functions/reference/max.md
index 4bb2145d683..12c8800ef7f 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/max.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/max.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/max
-sidebar_position: 3
+sidebar_position: 162
title: max
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
index db99b900a3e..c65e31114ff 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/maxintersections
-sidebar_position: 360
+sidebar_position: 163
title: maxIntersections
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxintersectionsposition.md b/docs/en/sql-reference/aggregate-functions/reference/maxintersectionsposition.md
index 7dd63f09316..d5c2b0bd3c2 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/maxintersectionsposition.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/maxintersectionsposition.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/maxintersectionsposition
-sidebar_position: 361
+sidebar_position: 164
title: maxIntersectionsPosition
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxmap.md b/docs/en/sql-reference/aggregate-functions/reference/maxmap.md
index 4d8c67e1b90..c9c6913249c 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/maxmap.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/maxmap.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/maxmap
-sidebar_position: 143
+sidebar_position: 165
---
# maxMap
diff --git a/docs/en/sql-reference/aggregate-functions/reference/meanztest.md b/docs/en/sql-reference/aggregate-functions/reference/meanztest.md
index 1cf2bebf26f..19afb5ae742 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/meanztest.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/meanztest.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/meanztest
-sidebar_position: 303
+sidebar_position: 166
sidebar_label: meanZTest
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/median.md b/docs/en/sql-reference/aggregate-functions/reference/median.md
index 2a166c83dad..dcf174254ac 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/median.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/median.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/median
-sidebar_position: 212
+sidebar_position: 167
---
# median
diff --git a/docs/en/sql-reference/aggregate-functions/reference/min.md b/docs/en/sql-reference/aggregate-functions/reference/min.md
index cca515b76e8..6bfcaf020c8 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/min.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/min.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/min
-sidebar_position: 2
+sidebar_position: 168
title: min
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/minmap.md b/docs/en/sql-reference/aggregate-functions/reference/minmap.md
index 5436e1fc6a6..b1fbb9e49f3 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/minmap.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/minmap.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/minmap
-sidebar_position: 142
+sidebar_position: 169
---
# minMap
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantile.md b/docs/en/sql-reference/aggregate-functions/reference/quantile.md
index 91b6b1b0d80..d5278125cbc 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantile.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantile.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantile
-sidebar_position: 200
+sidebar_position: 170
---
# quantile
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantileGK.md b/docs/en/sql-reference/aggregate-functions/reference/quantileGK.md
index 7352781d126..9582f264a6f 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantileGK.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantileGK.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantileGK
-sidebar_position: 204
+sidebar_position: 175
---
# quantileGK
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md
index 4377f2f1b17..4469438db6a 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantilebfloat16
-sidebar_position: 209
+sidebar_position: 171
title: quantileBFloat16
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantileddsketch.md b/docs/en/sql-reference/aggregate-functions/reference/quantileddsketch.md
index f9acd2e20cb..fc9db7ef08d 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantileddsketch.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantileddsketch.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantileddsketch
-sidebar_position: 211
+sidebar_position: 171
title: quantileDD
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiledeterministic.md b/docs/en/sql-reference/aggregate-functions/reference/quantiledeterministic.md
index 7235c47da70..0ac4b5e3a51 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantiledeterministic.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantiledeterministic.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantiledeterministic
-sidebar_position: 206
+sidebar_position: 172
---
# quantileDeterministic
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantileexact.md b/docs/en/sql-reference/aggregate-functions/reference/quantileexact.md
index d7d7413c283..46873bcd2b6 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantileexact.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantileexact.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantileexact
-sidebar_position: 202
+sidebar_position: 173
---
# quantileExact Functions
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantileexactweighted.md b/docs/en/sql-reference/aggregate-functions/reference/quantileexactweighted.md
index 34def8d7411..4ce212888c4 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantileexactweighted.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantileexactweighted.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantileexactweighted
-sidebar_position: 203
+sidebar_position: 174
---
# quantileExactWeighted
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md b/docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md
index 41d2627fb7b..9eb4fde6102 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantileInterpolatedWeighted
-sidebar_position: 203
+sidebar_position: 176
---
# quantileInterpolatedWeighted
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md
index 856d447ac13..e2c3295221d 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantiles
-sidebar_position: 201
+sidebar_position: 177
---
# quantiles Functions
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiletdigest.md b/docs/en/sql-reference/aggregate-functions/reference/quantiletdigest.md
index 796e87b02d8..ece54ca24ab 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantiletdigest.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantiletdigest.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantiletdigest
-sidebar_position: 207
+sidebar_position: 178
---
# quantileTDigest
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md b/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md
index b3e21e0e69e..7f8f7f53a97 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantiletdigestweighted
-sidebar_position: 208
+sidebar_position: 179
---
# quantileTDigestWeighted
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiletiming.md b/docs/en/sql-reference/aggregate-functions/reference/quantiletiming.md
index b5b1c8a0c01..78050fe5b5e 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantiletiming.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantiletiming.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantiletiming
-sidebar_position: 204
+sidebar_position: 180
---
# quantileTiming
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiletimingweighted.md b/docs/en/sql-reference/aggregate-functions/reference/quantiletimingweighted.md
index df483aac01e..c5fff0825c3 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantiletimingweighted.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantiletimingweighted.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/quantiletimingweighted
-sidebar_position: 205
+sidebar_position: 181
---
# quantileTimingWeighted
diff --git a/docs/en/sql-reference/aggregate-functions/reference/rankCorr.md b/docs/en/sql-reference/aggregate-functions/reference/rankCorr.md
index 27f2dd124e4..eb995923d97 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/rankCorr.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/rankCorr.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/rankCorr
-sidebar_position: 145
+sidebar_position: 182
---
# rankCorr
diff --git a/docs/en/sql-reference/aggregate-functions/reference/simplelinearregression.md b/docs/en/sql-reference/aggregate-functions/reference/simplelinearregression.md
index ea3dbff8691..2aebccfdc53 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/simplelinearregression.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/simplelinearregression.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/simplelinearregression
-sidebar_position: 220
+sidebar_position: 183
---
# simpleLinearRegression
diff --git a/docs/en/sql-reference/aggregate-functions/reference/singlevalueornull.md b/docs/en/sql-reference/aggregate-functions/reference/singlevalueornull.md
index e39af77059a..21344b58ba6 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/singlevalueornull.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/singlevalueornull.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/singlevalueornull
-sidebar_position: 220
+sidebar_position: 184
---
# singleValueOrNull
diff --git a/docs/en/sql-reference/aggregate-functions/reference/skewpop.md b/docs/en/sql-reference/aggregate-functions/reference/skewpop.md
index 379fdcfa7c2..58ea33edb81 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/skewpop.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/skewpop.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/skewpop
-sidebar_position: 150
+sidebar_position: 185
---
# skewPop
diff --git a/docs/en/sql-reference/aggregate-functions/reference/skewsamp.md b/docs/en/sql-reference/aggregate-functions/reference/skewsamp.md
index 9e64b186db3..9c32a0183ef 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/skewsamp.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/skewsamp.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/skewsamp
-sidebar_position: 151
+sidebar_position: 186
---
# skewSamp
diff --git a/docs/en/sql-reference/aggregate-functions/reference/sparkbar.md b/docs/en/sql-reference/aggregate-functions/reference/sparkbar.md
index 62edc221858..8791847ead0 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/sparkbar.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/sparkbar.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/sparkbar
-sidebar_position: 311
+sidebar_position: 187
sidebar_label: sparkbar
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/stddevpop.md b/docs/en/sql-reference/aggregate-functions/reference/stddevpop.md
index d2406197ecc..e52a442d76a 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/stddevpop.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/stddevpop.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/stddevpop
-sidebar_position: 30
+sidebar_position: 188
---
# stddevPop
@@ -25,7 +25,7 @@ stddevPop(x)
**Returned value**
-Square root of standard deviation of `x`. [Float64](../../data-types/float.md).
+- Square root of standard deviation of `x`. [Float64](../../data-types/float.md).
**Example**
diff --git a/docs/en/sql-reference/aggregate-functions/reference/stddevpopstable.md b/docs/en/sql-reference/aggregate-functions/reference/stddevpopstable.md
index a8ad5956ae8..2051ce7b125 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/stddevpopstable.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/stddevpopstable.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/stddevpopstable
-sidebar_position: 30
+sidebar_position: 189
---
# stddevPopStable
diff --git a/docs/en/sql-reference/aggregate-functions/reference/stddevsamp.md b/docs/en/sql-reference/aggregate-functions/reference/stddevsamp.md
index cf8b9b20d63..e2cad40b267 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/stddevsamp.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/stddevsamp.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/stddevsamp
-sidebar_position: 31
+sidebar_position: 190
---
# stddevSamp
diff --git a/docs/en/sql-reference/aggregate-functions/reference/stddevsampstable.md b/docs/en/sql-reference/aggregate-functions/reference/stddevsampstable.md
index 9ae1f5f8411..205e10cced5 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/stddevsampstable.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/stddevsampstable.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/stddevsampstable
-sidebar_position: 31
+sidebar_position: 191
---
# stddevSampStable
diff --git a/docs/en/sql-reference/aggregate-functions/reference/stochasticlinearregression.md b/docs/en/sql-reference/aggregate-functions/reference/stochasticlinearregression.md
index ddac82a0977..6cc5cbd8fe1 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/stochasticlinearregression.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/stochasticlinearregression.md
@@ -1,9 +1,9 @@
---
slug: /en/sql-reference/aggregate-functions/reference/stochasticlinearregression
-sidebar_position: 221
+sidebar_position: 192
---
-# stochasticLinearRegression
+# stochasticLinearRegression {#agg_functions_stochasticlinearregression_parameters}
This function implements stochastic linear regression. It supports custom parameters for learning rate, L2 regularization coefficient, mini-batch size, and has a few methods for updating weights ([Adam](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) (used by default), [simple SGD](https://en.wikipedia.org/wiki/Stochastic_gradient_descent), [Momentum](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum), and [Nesterov](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)).
@@ -72,5 +72,5 @@ The query will return a column of predicted values. Note that first argument of
**See Also**
-- [stochasticLogisticRegression](../../../sql-reference/aggregate-functions/reference/stochasticlogisticregression.md#agg_functions-stochasticlogisticregression)
+- [stochasticLogisticRegression](../../../sql-reference/aggregate-functions/reference/stochasticlogisticregression.md#stochasticlogisticregression)
- [Difference between linear and logistic regressions](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression)
diff --git a/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression.md b/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression.md
index 0a040689681..dca452a1702 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/stochasticlogisticregression
-sidebar_position: 222
+sidebar_position: 193
---
# stochasticLogisticRegression
@@ -11,7 +11,7 @@ This function implements stochastic logistic regression. It can be used for bina
Parameters are exactly the same as in stochasticLinearRegression:
`learning rate`, `l2 regularization coefficient`, `mini-batch size`, `method for updating weights`.
-For more information see [parameters](#agg_functions-stochasticlinearregression-parameters).
+For more information see [parameters](../reference/stochasticlinearregression.md/#parameters).
``` text
stochasticLogisticRegression(1.0, 1.0, 10, 'SGD')
diff --git a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md
index fa320b4e336..1605e8efa13 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/studentttest
-sidebar_position: 300
+sidebar_position: 194
sidebar_label: studentTTest
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/sum.md b/docs/en/sql-reference/aggregate-functions/reference/sum.md
index a33a99f63e6..19636f003c7 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/sum.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/sum.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/sum
-sidebar_position: 4
+sidebar_position: 195
---
# sum
diff --git a/docs/en/sql-reference/aggregate-functions/reference/sumcount.md b/docs/en/sql-reference/aggregate-functions/reference/sumcount.md
index a59b87022d6..ff4ddcec142 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/sumcount.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/sumcount.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/sumcount
-sidebar_position: 144
+sidebar_position: 196
title: sumCount
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/sumkahan.md b/docs/en/sql-reference/aggregate-functions/reference/sumkahan.md
index 1a729b18b42..ed58b3c3369 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/sumkahan.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/sumkahan.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/sumkahan
-sidebar_position: 145
+sidebar_position: 197
title: sumKahan
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/summap.md b/docs/en/sql-reference/aggregate-functions/reference/summap.md
index fd3f095511b..4ff937f1e4f 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/summap.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/summap.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/summap
-sidebar_position: 141
+sidebar_position: 198
---
# sumMap
diff --git a/docs/en/sql-reference/aggregate-functions/reference/summapwithoverflow.md b/docs/en/sql-reference/aggregate-functions/reference/summapwithoverflow.md
index 7c0aa31e459..e36818e2ab8 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/summapwithoverflow.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/summapwithoverflow.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/summapwithoverflow
-sidebar_position: 141
+sidebar_position: 199
---
# sumMapWithOverflow
diff --git a/docs/en/sql-reference/aggregate-functions/reference/sumwithoverflow.md b/docs/en/sql-reference/aggregate-functions/reference/sumwithoverflow.md
index a120eafe738..5fe3cb7de8e 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/sumwithoverflow.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/sumwithoverflow.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/sumwithoverflow
-sidebar_position: 140
+sidebar_position: 200
---
# sumWithOverflow
diff --git a/docs/en/sql-reference/aggregate-functions/reference/theilsu.md b/docs/en/sql-reference/aggregate-functions/reference/theilsu.md
index ef19438a53a..73b063cf965 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/theilsu.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/theilsu.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/theilsu
-sidebar_position: 353
+sidebar_position: 201
---
# theilsU
diff --git a/docs/en/sql-reference/aggregate-functions/reference/topk.md b/docs/en/sql-reference/aggregate-functions/reference/topk.md
index dd4b2251a8a..695e9b1d7d8 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/topk.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/topk.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/topk
-sidebar_position: 108
+sidebar_position: 202
---
# topK
diff --git a/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md b/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md
index d2a469828fc..148a8b6ea18 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/topkweighted
-sidebar_position: 109
+sidebar_position: 203
---
# topKWeighted
diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniq.md b/docs/en/sql-reference/aggregate-functions/reference/uniq.md
index b1c8336630b..c1dc6a29e58 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/uniq.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/uniq.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/uniq
-sidebar_position: 190
+sidebar_position: 204
---
# uniq
diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md
index 18f44d2fcc4..70bb4463140 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/uniqcombined
-sidebar_position: 192
+sidebar_position: 205
---
# uniqCombined
diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md
index b6e09bcaae3..014984f6291 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/uniqcombined64
-sidebar_position: 193
+sidebar_position: 206
---
# uniqCombined64
diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqexact.md b/docs/en/sql-reference/aggregate-functions/reference/uniqexact.md
index fd68a464881..da4d4aa9588 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/uniqexact.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/uniqexact.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/uniqexact
-sidebar_position: 191
+sidebar_position: 207
---
# uniqExact
diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md b/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md
index 8594ebb3782..78d84edf1be 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/uniqhll12
-sidebar_position: 194
+sidebar_position: 208
---
# uniqHLL12
diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqthetasketch.md b/docs/en/sql-reference/aggregate-functions/reference/uniqthetasketch.md
index 45970f144cb..fbae42117ee 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/uniqthetasketch.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/uniqthetasketch.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/uniqthetasketch
-sidebar_position: 195
+sidebar_position: 209
title: uniqTheta
---
diff --git a/docs/en/sql-reference/aggregate-functions/reference/varpop.md b/docs/en/sql-reference/aggregate-functions/reference/varpop.md
index fcabeb4c6a8..182e830f19f 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/varpop.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/varpop.md
@@ -1,33 +1,28 @@
---
title: "varPop"
-slug: "/en/sql-reference/aggregate-functions/reference/varpop"
-sidebar_position: 32
+slug: "/en/sql-reference/aggregate-functions/reference/varPop"
+sidebar_position: 210
---
-This page covers the `varPop` and `varPopStable` functions available in ClickHouse.
-
## varPop
-Calculates the population covariance between two data columns. The population covariance measures the degree to which two variables vary together. Calculates the amount `Σ((x - x̅)^2) / n`, where `n` is the sample size and `x̅`is the average value of `x`.
+Calculates the population variance.
**Syntax**
```sql
-covarPop(x, y)
+varPop(x)
```
+Alias: `VAR_POP`.
+
**Parameters**
-- `x`: The first data column. [Numeric](../../../native-protocol/columns.md)
-- `y`: The second data column. [Numeric](../../../native-protocol/columns.md)
+- `x`: Population of values to find the population variance of. [(U)Int*](../../data-types/int-uint.md), [Float*](../../data-types/float.md), [Decimal*](../../data-types/decimal.md).
**Returned value**
-Returns an integer of type `Float64`.
-
-**Implementation details**
-
-This function uses a numerically unstable algorithm. If you need numerical stability in calculations, use the slower but more stable [`varPopStable` function](#varPopStable).
+- Returns the population variance of `x`. [`Float64`](../../data-types/float.md).
**Example**
@@ -37,69 +32,21 @@ Query:
DROP TABLE IF EXISTS test_data;
CREATE TABLE test_data
(
- x Int32,
- y Int32
+ x UInt8,
)
ENGINE = Memory;
-INSERT INTO test_data VALUES (1, 2), (2, 3), (3, 5), (4, 6), (5, 8);
+INSERT INTO test_data VALUES (3), (3), (3), (4), (4), (5), (5), (7), (11), (15);
SELECT
- covarPop(x, y) AS covar_pop
+ varPop(x) AS var_pop
FROM test_data;
```
Result:
```response
-3
-```
-
-## varPopStable
-
-Calculates population covariance between two data columns using a stable, numerically accurate method to calculate the variance. This function is designed to provide reliable results even with large datasets or values that might cause numerical instability in other implementations.
-
-**Syntax**
-
-```sql
-covarPopStable(x, y)
-```
-
-**Parameters**
-
-- `x`: The first data column. [String literal](../../syntax#syntax-string-literal)
-- `y`: The second data column. [Expression](../../syntax#syntax-expressions)
-
-**Returned value**
-
-Returns an integer of type `Float64`.
-
-**Implementation details**
-
-Unlike [`varPop()`](#varPop), this function uses a stable, numerically accurate algorithm to calculate the population variance to avoid issues like catastrophic cancellation or loss of precision. This function also handles `NaN` and `Inf` values correctly, excluding them from calculations.
-
-**Example**
-
-Query:
-
-```sql
-DROP TABLE IF EXISTS test_data;
-CREATE TABLE test_data
-(
- x Int32,
- y Int32
-)
-ENGINE = Memory;
-
-INSERT INTO test_data VALUES (1, 2), (2, 9), (9, 5), (4, 6), (5, 8);
-
-SELECT
- covarPopStable(x, y) AS covar_pop_stable
-FROM test_data;
-```
-
-Result:
-
-```response
-0.5999999999999999
+┌─var_pop─┐
+│ 14.4 │
+└─────────┘
```
diff --git a/docs/en/sql-reference/aggregate-functions/reference/varpopstable.md b/docs/en/sql-reference/aggregate-functions/reference/varpopstable.md
new file mode 100644
index 00000000000..68037a5a533
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/varpopstable.md
@@ -0,0 +1,52 @@
+---
+title: "varPopStable"
+slug: "/en/sql-reference/aggregate-functions/reference/varpopstable"
+sidebar_position: 211
+---
+
+## varPopStable
+
+Returns the population variance. Unlike [`varPop`](../reference/varpop.md), this function uses a [numerically stable](https://en.wikipedia.org/wiki/Numerical_stability) algorithm. It works slower but provides a lower computational error.
+
+**Syntax**
+
+```sql
+varPopStable(x)
+```
+
+Alias: `VAR_POP_STABLE`.
+
+**Parameters**
+
+- `x`: Population of values to find the population variance of. [(U)Int*](../../data-types/int-uint.md), [Float*](../../data-types/float.md), [Decimal*](../../data-types/decimal.md).
+
+**Returned value**
+
+- Returns the population variance of `x`. [Float64](../../data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+DROP TABLE IF EXISTS test_data;
+CREATE TABLE test_data
+(
+ x UInt8,
+)
+ENGINE = Memory;
+
+INSERT INTO test_data VALUES (3),(3),(3),(4),(4),(5),(5),(7),(11),(15);
+
+SELECT
+ varPopStable(x) AS var_pop_stable
+FROM test_data;
+```
+
+Result:
+
+```response
+┌─var_pop_stable─┐
+│ 14.4 │
+└────────────────┘
+```
diff --git a/docs/en/sql-reference/aggregate-functions/reference/varsamp.md b/docs/en/sql-reference/aggregate-functions/reference/varsamp.md
index be669a16ae8..87a97c15dd8 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/varsamp.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/varsamp.md
@@ -1,11 +1,9 @@
---
title: "varSamp"
-slug: /en/sql-reference/aggregate-functions/reference/varsamp
-sidebar_position: 33
+slug: /en/sql-reference/aggregate-functions/reference/varSamp
+sidebar_position: 212
---
-This page contains information on the `varSamp` and `varSampStable` ClickHouse functions.
-
## varSamp
Calculate the sample variance of a data set.
@@ -13,24 +11,27 @@ Calculate the sample variance of a data set.
**Syntax**
```sql
-varSamp(expr)
+varSamp(x)
```
+Alias: `VAR_SAMP`.
+
**Parameters**
-- `expr`: An expression representing the data set for which you want to calculate the sample variance. [Expression](../../syntax#syntax-expressions)
+- `x`: The population for which you want to calculate the sample variance. [(U)Int*](../../data-types/int-uint.md), [Float*](../../data-types/float.md), [Decimal*](../../data-types/decimal.md).
**Returned value**
-Returns a Float64 value representing the sample variance of the input data set.
+
+- Returns the sample variance of the input data set `x`. [Float64](../../data-types/float.md).
**Implementation details**
-The `varSamp()` function calculates the sample variance using the following formula:
+The `varSamp` function calculates the sample variance using the following formula:
-```plaintext
-∑(x - mean(x))^2 / (n - 1)
-```
+$$
+\sum\frac{(x - \text{mean}(x))^2}{(n - 1)}
+$$
Where:
@@ -38,91 +39,29 @@ Where:
- `mean(x)` is the arithmetic mean of the data set.
- `n` is the number of data points in the data set.
-The function assumes that the input data set represents a sample from a larger population. If you want to calculate the variance of the entire population (when you have the complete data set), you should use the [`varPop()` function](./varpop#varpop) instead.
-
-This function uses a numerically unstable algorithm. If you need numerical stability in calculations, use the slower but more stable [`varSampStable` function](#varSampStable).
+The function assumes that the input data set represents a sample from a larger population. If you want to calculate the variance of the entire population (when you have the complete data set), you should use [`varPop`](../reference/varpop.md) instead.
**Example**
Query:
```sql
-CREATE TABLE example_table
+DROP TABLE IF EXISTS test_data;
+CREATE TABLE test_data
(
- id UInt64,
- value Float64
+ x Float64
)
-ENGINE = MergeTree
-ORDER BY id;
+ENGINE = Memory;
-INSERT INTO example_table VALUES (1, 10.5), (2, 12.3), (3, 9.8), (4, 11.2), (5, 10.7);
+INSERT INTO test_data VALUES (10.5), (12.3), (9.8), (11.2), (10.7);
-SELECT varSamp(value) FROM example_table;
+SELECT round(varSamp(x),3) AS var_samp FROM test_data;
```
Response:
```response
-0.8650000000000091
+┌─var_samp─┐
+│ 0.865 │
+└──────────┘
```
-
-## varSampStable
-
-Calculate the sample variance of a data set using a numerically stable algorithm.
-
-**Syntax**
-
-```sql
-varSampStable(expr)
-```
-
-**Parameters**
-
-- `expr`: An expression representing the data set for which you want to calculate the sample variance. [Expression](../../syntax#syntax-expressions)
-
-**Returned value**
-
-The `varSampStable()` function returns a Float64 value representing the sample variance of the input data set.
-
-**Implementation details**
-
-The `varSampStable()` function calculates the sample variance using the same formula as the [`varSamp()`](#varSamp function):
-
-```plaintext
-∑(x - mean(x))^2 / (n - 1)
-```
-
-Where:
-- `x` is each individual data point in the data set.
-- `mean(x)` is the arithmetic mean of the data set.
-- `n` is the number of data points in the data set.
-
-The difference between `varSampStable()` and `varSamp()` is that `varSampStable()` is designed to provide a more deterministic and stable result when dealing with floating-point arithmetic. It uses an algorithm that minimizes the accumulation of rounding errors, which can be particularly important when dealing with large data sets or data with a wide range of values.
-
-Like `varSamp()`, the `varSampStable()` function assumes that the input data set represents a sample from a larger population. If you want to calculate the variance of the entire population (when you have the complete data set), you should use the [`varPopStable()` function](./varpop#varpopstable) instead.
-
-**Example**
-
-Query:
-
-```sql
-CREATE TABLE example_table
-(
- id UInt64,
- value Float64
-)
-ENGINE = MergeTree
-ORDER BY id;
-
-INSERT INTO example_table VALUES (1, 10.5), (2, 12.3), (3, 9.8), (4, 11.2), (5, 10.7);
-
-SELECT varSampStable(value) FROM example_table;
-```
-
-Response:
-
-```response
-0.865
-```
-
-This query calculates the sample variance of the `value` column in the `example_table` using the `varSampStable()` function. The result shows that the sample variance of the values `[10.5, 12.3, 9.8, 11.2, 10.7]` is approximately 0.865, which may differ slightly from the result of `varSamp()` due to the more precise handling of floating-point arithmetic.
diff --git a/docs/en/sql-reference/aggregate-functions/reference/varsampstable.md b/docs/en/sql-reference/aggregate-functions/reference/varsampstable.md
new file mode 100644
index 00000000000..ebe647e1951
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/varsampstable.md
@@ -0,0 +1,63 @@
+---
+title: "varSampStable"
+slug: /en/sql-reference/aggregate-functions/reference/varsampstable
+sidebar_position: 213
+---
+
+## varSampStable
+
+Calculate the sample variance of a data set. Unlike [`varSamp`](../reference/varsamp.md), this function uses a numerically stable algorithm. It works slower but provides a lower computational error.
+
+**Syntax**
+
+```sql
+varSampStable(x)
+```
+
+Alias: `VAR_SAMP_STABLE`
+
+**Parameters**
+
+- `x`: The population for which you want to calculate the sample variance. [(U)Int*](../../data-types/int-uint.md), [Float*](../../data-types/float.md), [Decimal*](../../data-types/decimal.md).
+
+**Returned value**
+
+- Returns the sample variance of the input data set. [Float64](../../data-types/float.md).
+
+**Implementation details**
+
+The `varSampStable` function calculates the sample variance using the same formula as the [`varSamp`](../reference/varsamp.md):
+
+$$
+\sum\frac{(x - \text{mean}(x))^2}{(n - 1)}
+$$
+
+Where:
+- `x` is each individual data point in the data set.
+- `mean(x)` is the arithmetic mean of the data set.
+- `n` is the number of data points in the data set.
+
+**Example**
+
+Query:
+
+```sql
+DROP TABLE IF EXISTS test_data;
+CREATE TABLE test_data
+(
+ x Float64
+)
+ENGINE = Memory;
+
+INSERT INTO test_data VALUES (10.5), (12.3), (9.8), (11.2), (10.7);
+
+SELECT round(varSampStable(x),3) AS var_samp_stable FROM test_data;
+```
+
+Response:
+
+```response
+┌─var_samp_stable─┐
+│ 0.865 │
+└─────────────────┘
+```
diff --git a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md
index 4f1085e65b4..296b70f758e 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/aggregate-functions/reference/welchttest
-sidebar_position: 301
+sidebar_position: 214
sidebar_label: welchTTest
---
diff --git a/docs/en/sql-reference/data-types/datetime.md b/docs/en/sql-reference/data-types/datetime.md
index ac9a72c2641..250e766f2b7 100644
--- a/docs/en/sql-reference/data-types/datetime.md
+++ b/docs/en/sql-reference/data-types/datetime.md
@@ -137,7 +137,7 @@ If the time transition (due to daylight saving time or for other reasons) was pe
Non-monotonic calendar dates. For example, in Happy Valley - Goose Bay, the time was transitioned one hour backwards at 00:01:00 7 Nov 2010 (one minute after midnight). So after 6th Nov has ended, people observed a whole one minute of 7th Nov, then time was changed back to 23:01 6th Nov and after another 59 minutes the 7th Nov started again. ClickHouse does not (yet) support this kind of fun. During these days the results of time processing functions may be slightly incorrect.
-Similar issue exists for Casey Antarctic station in year 2010. They changed time three hours back at 5 Mar, 02:00. If you are working in antarctic station, please don't afraid to use ClickHouse. Just make sure you set timezone to UTC or be aware of inaccuracies.
+Similar issue exists for Casey Antarctic station in year 2010. They changed time three hours back at 5 Mar, 02:00. If you are working in antarctic station, please don't be afraid to use ClickHouse. Just make sure you set timezone to UTC or be aware of inaccuracies.
Time shifts for multiple days. Some pacific islands changed their timezone offset from UTC+14 to UTC-12. That's alright but some inaccuracies may present if you do calculations with their timezone for historical time points at the days of conversion.
diff --git a/docs/en/sql-reference/data-types/float.md b/docs/en/sql-reference/data-types/float.md
index 23131d5b4fe..3c789076c1e 100644
--- a/docs/en/sql-reference/data-types/float.md
+++ b/docs/en/sql-reference/data-types/float.md
@@ -7,33 +7,43 @@ sidebar_label: Float32, Float64
# Float32, Float64
:::note
-If you need accurate calculations, in particular if you work with financial or business data requiring a high precision you should consider using Decimal instead. Floats might lead to inaccurate results as illustrated below:
+If you need accurate calculations, in particular if you work with financial or business data requiring a high precision, you should consider using [Decimal](../data-types/decimal.md) instead.
-```
+[Floating Point Numbers](https://en.wikipedia.org/wiki/IEEE_754) might lead to inaccurate results as illustrated below:
+
+```sql
CREATE TABLE IF NOT EXISTS float_vs_decimal
(
my_float Float64,
my_decimal Decimal64(3)
-)Engine=MergeTree ORDER BY tuple()
-
-INSERT INTO float_vs_decimal SELECT round(randCanonical(), 3) AS res, res FROM system.numbers LIMIT 1000000; # Generate 1 000 000 random number with 2 decimal places and store them as a float and as a decimal
+)
+Engine=MergeTree
+ORDER BY tuple();
+# Generate 1 000 000 random numbers with 2 decimal places and store them as a float and as a decimal
+INSERT INTO float_vs_decimal SELECT round(randCanonical(), 3) AS res, res FROM system.numbers LIMIT 1000000;
+```
+```
SELECT sum(my_float), sum(my_decimal) FROM float_vs_decimal;
-> 500279.56300000014 500279.563
+
+┌──────sum(my_float)─┬─sum(my_decimal)─┐
+│ 499693.60500000004 │ 499693.605 │
+└────────────────────┴─────────────────┘
SELECT sumKahan(my_float), sumKahan(my_decimal) FROM float_vs_decimal;
-> 500279.563 500279.563
+
+┌─sumKahan(my_float)─┬─sumKahan(my_decimal)─┐
+│ 499693.605 │ 499693.605 │
+└────────────────────┴──────────────────────┘
```
:::
-[Floating point numbers](https://en.wikipedia.org/wiki/IEEE_754).
-
-Types are equivalent to types of C:
+The equivalent types in ClickHouse and in C are given below:
- `Float32` — `float`.
- `Float64` — `double`.
-Aliases:
+Float types in ClickHouse have the following aliases:
- `Float32` — `FLOAT`, `REAL`, `SINGLE`.
- `Float64` — `DOUBLE`, `DOUBLE PRECISION`.
diff --git a/docs/en/sql-reference/data-types/geo.md b/docs/en/sql-reference/data-types/geo.md
index 7e3c32b3451..7ffc7447d96 100644
--- a/docs/en/sql-reference/data-types/geo.md
+++ b/docs/en/sql-reference/data-types/geo.md
@@ -33,7 +33,7 @@ Result:
## Ring
-`Ring` is a simple polygon without holes stored as an array of points: [Array](array.md)([Point](#point-data-type)).
+`Ring` is a simple polygon without holes stored as an array of points: [Array](array.md)([Point](#point)).
**Example**
@@ -54,7 +54,7 @@ Result:
## Polygon
-`Polygon` is a polygon with holes stored as an array of rings: [Array](array.md)([Ring](#ring-data-type)). First element of outer array is the outer shape of polygon and all the following elements are holes.
+`Polygon` is a polygon with holes stored as an array of rings: [Array](array.md)([Ring](#ring)). First element of outer array is the outer shape of polygon and all the following elements are holes.
**Example**
@@ -76,7 +76,7 @@ Result:
## MultiPolygon
-`MultiPolygon` consists of multiple polygons and is stored as an array of polygons: [Array](array.md)([Polygon](#polygon-data-type)).
+`MultiPolygon` consists of multiple polygons and is stored as an array of polygons: [Array](array.md)([Polygon](#polygon)).
**Example**
diff --git a/docs/en/sql-reference/data-types/json.md b/docs/en/sql-reference/data-types/json.md
index 39e37abad82..c29be2cff58 100644
--- a/docs/en/sql-reference/data-types/json.md
+++ b/docs/en/sql-reference/data-types/json.md
@@ -1,24 +1,20 @@
---
-slug: /en/sql-reference/data-types/json
+slug: /en/sql-reference/data-types/object-data-type
sidebar_position: 26
-sidebar_label: JSON
+sidebar_label: Object Data Type
+keywords: [object, data type]
---
-# JSON
+# Object Data Type
:::note
-This feature is experimental and is not production-ready. If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-ingestion/data-formats/json.md) instead.
+This feature is not production-ready and is now deprecated. If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-ingestion/data-formats/json) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864)
:::
Stores JavaScript Object Notation (JSON) documents in a single column.
`JSON` is an alias for `Object('json')`.
-:::note
-The JSON data type is an obsolete feature. Do not use it.
-If you want to use it, set `allow_experimental_object_type = 1`.
-:::
-
## Example
**Example 1**
@@ -49,7 +45,7 @@ SELECT o.a, o.b.c, o.b.d[3] FROM json
**Example 2**
-To be able to create an ordered `MergeTree` family table the sorting key has to be extracted into its column. For example, to insert a file of compressed HTTP access logs in JSON format:
+To be able to create an ordered `MergeTree` family table, the sorting key has to be extracted into its column. For example, to insert a file of compressed HTTP access logs in JSON format:
```sql
CREATE TABLE logs
@@ -69,7 +65,7 @@ FROM file('access.json.gz', JSONAsString)
## Displaying JSON columns
-When displaying a `JSON` column ClickHouse only shows the field values by default (because internally, it is represented as a tuple). You can display the field names as well by setting `output_format_json_named_tuples_as_objects = 1`:
+When displaying a `JSON` column, ClickHouse only shows the field values by default (because internally, it is represented as a tuple). You can also display the field names by setting `output_format_json_named_tuples_as_objects = 1`:
```sql
SET output_format_json_named_tuples_as_objects = 1
@@ -83,4 +79,5 @@ SELECT * FROM json FORMAT JSONEachRow
## Related Content
+- [Using JSON in ClickHouse](/docs/en/integrations/data-formats/json)
- [Getting Data Into ClickHouse - Part 2 - A JSON detour](https://clickhouse.com/blog/getting-data-into-clickhouse-part-2-json)
diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 080de94f8b7..4c7421d57c0 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -16,7 +16,7 @@ ClickHouse supports special functions for working with dictionaries that can be
ClickHouse supports:
- Dictionaries with a [set of functions](../../sql-reference/functions/ext-dict-functions.md).
-- [Embedded dictionaries](#embedded_dictionaries) with a specific [set of functions](../../sql-reference/functions/ym-dict-functions.md).
+- [Embedded dictionaries](#embedded-dictionaries) with a specific [set of functions](../../sql-reference/functions/ym-dict-functions.md).
:::tip Tutorial
@@ -82,7 +82,7 @@ You can [configure](#configuring-a-dictionary) any number of dictionaries in the
You can convert values for a small dictionary by describing it in a `SELECT` query (see the [transform](../../sql-reference/functions/other-functions.md) function). This functionality is not related to dictionaries.
:::
-## Configuring a Dictionary {#configuring-a-dictionary}
+## Configuring a Dictionary
@@ -123,7 +123,7 @@ LAYOUT(...) -- Memory layout configuration
LIFETIME(...) -- Lifetime of dictionary in memory
```
-## Storing Dictionaries in Memory {#storing-dictionaries-in-memory}
+## Storing Dictionaries in Memory
There are a variety of ways to store dictionaries in memory.
@@ -415,7 +415,7 @@ or
LAYOUT(COMPLEX_KEY_HASHED_ARRAY([SHARDS 1]))
```
-### range_hashed {#range_hashed}
+### range_hashed
The dictionary is stored in memory in the form of a hash table with an ordered array of ranges and their corresponding values.
@@ -679,7 +679,7 @@ When searching for a dictionary, the cache is searched first. For each block of
If keys are not found in dictionary, then update cache task is created and added into update queue. Update queue properties can be controlled with settings `max_update_queue_size`, `update_queue_push_timeout_milliseconds`, `query_wait_timeout_milliseconds`, `max_threads_for_updates`.
-For cache dictionaries, the expiration [lifetime](#dictionary-updates) of data in the cache can be set. If more time than `lifetime` has passed since loading the data in a cell, the cell’s value is not used and key becomes expired. The key is re-requested the next time it needs to be used. This behaviour can be configured with setting `allow_read_expired_keys`.
+For cache dictionaries, the expiration [lifetime](#refreshing-dictionary-data-using-lifetime) of data in the cache can be set. If more time than `lifetime` has passed since loading the data in a cell, the cell’s value is not used and key becomes expired. The key is re-requested the next time it needs to be used. This behaviour can be configured with setting `allow_read_expired_keys`.
This is the least effective of all the ways to store dictionaries. The speed of the cache depends strongly on correct settings and the usage scenario. A cache type dictionary performs well only when the hit rates are high enough (recommended 99% and higher). You can view the average hit rate in the [system.dictionaries](../../operations/system-tables/dictionaries.md) table.
@@ -899,7 +899,7 @@ Other types are not supported yet. The function returns the attribute for the pr
Data must completely fit into RAM.
-## Refreshing dictionary data using LIFETIME {#lifetime}
+## Refreshing dictionary data using LIFETIME
ClickHouse periodically updates dictionaries based on the `LIFETIME` tag (defined in seconds). `LIFETIME` is the update interval for fully downloaded dictionaries and the invalidation interval for cached dictionaries.
@@ -1031,7 +1031,7 @@ SOURCE(CLICKHOUSE(... update_field 'added_time' update_lag 15))
...
```
-## Dictionary Sources {#dictionary-sources}
+## Dictionary Sources
@@ -1065,7 +1065,7 @@ SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration
The source is configured in the `source` section.
-For source types [Local file](#local_file), [Executable file](#executable), [HTTP(s)](#https), [ClickHouse](#clickhouse)
+For source types [Local file](#local-file), [Executable file](#executable-file), [HTTP(s)](#https), [ClickHouse](#clickhouse)
optional settings are available:
``` xml
@@ -1089,10 +1089,10 @@ SETTINGS(format_csv_allow_single_quotes = 0)
Types of sources (`source_type`):
-- [Local file](#local_file)
-- [Executable File](#executable)
-- [Executable Pool](#executable_pool)
-- [HTTP(S)](#http)
+- [Local file](#local-file)
+- [Executable File](#executable-file)
+- [Executable Pool](#executable-pool)
+- [HTTP(S)](#https)
- DBMS
- [ODBC](#odbc)
- [MySQL](#mysql)
@@ -1102,7 +1102,7 @@ Types of sources (`source_type`):
- [Cassandra](#cassandra)
- [PostgreSQL](#postgresql)
-### Local File {#local_file}
+### Local File
Example of settings:
@@ -1132,9 +1132,9 @@ When a dictionary with source `FILE` is created via DDL command (`CREATE DICTION
- [Dictionary function](../../sql-reference/table-functions/dictionary.md#dictionary-function)
-### Executable File {#executable}
+### Executable File
-Working with executable files depends on [how the dictionary is stored in memory](#storig-dictionaries-in-memory). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request to the executable file’s STDIN. Otherwise, ClickHouse starts the executable file and treats its output as dictionary data.
+Working with executable files depends on [how the dictionary is stored in memory](#storing-dictionaries-in-memory). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request to the executable file’s STDIN. Otherwise, ClickHouse starts the executable file and treats its output as dictionary data.
Example of settings:
@@ -1161,7 +1161,7 @@ Setting fields:
That dictionary source can be configured only via XML configuration. Creating dictionaries with executable source via DDL is disabled; otherwise, the DB user would be able to execute arbitrary binaries on the ClickHouse node.
-### Executable Pool {#executable_pool}
+### Executable Pool
Executable pool allows loading data from pool of processes. This source does not work with dictionary layouts that need to load all data from source. Executable pool works if the dictionary [is stored](#ways-to-store-dictionaries-in-memory) using `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, or `complex_key_direct` layouts.
@@ -1196,9 +1196,9 @@ Setting fields:
That dictionary source can be configured only via XML configuration. Creating dictionaries with executable source via DDL is disabled, otherwise, the DB user would be able to execute arbitrary binary on ClickHouse node.
-### HTTP(S) {#https}
+### HTTP(S)
-Working with an HTTP(S) server depends on [how the dictionary is stored in memory](#storig-dictionaries-in-memory). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method.
+Working with an HTTP(S) server depends on [how the dictionary is stored in memory](#storing-dictionaries-in-memory). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method.
Example of settings:
@@ -1285,7 +1285,7 @@ Setting fields:
- `db` – Name of the database. Omit it if the database name is set in the `` parameters.
- `table` – Name of the table and schema if exists.
- `connection_string` – Connection string.
-- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Updating dictionaries](#dictionary-updates).
+- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Refreshing dictionary data using LIFETIME](#refreshing-dictionary-data-using-lifetime).
- `query` – The custom query. Optional parameter.
:::note
@@ -1575,7 +1575,7 @@ Setting fields:
- `where` – The selection criteria. The syntax for conditions is the same as for `WHERE` clause in MySQL, for example, `id > 10 AND id < 20`. Optional parameter.
-- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Updating dictionaries](#dictionary-updates).
+- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Refreshing dictionary data using LIFETIME](#refreshing-dictionary-data-using-lifetime).
- `fail_on_connection_loss` – The configuration parameter that controls behavior of the server on connection loss. If `true`, an exception is thrown immediately if the connection between client and server was lost. If `false`, the ClickHouse server retries to execute the query three times before throwing an exception. Note that retrying leads to increased response times. Default value: `false`.
@@ -1672,7 +1672,7 @@ Setting fields:
- `db` – Name of the database.
- `table` – Name of the table.
- `where` – The selection criteria. May be omitted.
-- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Updating dictionaries](#dictionary-updates).
+- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Refreshing dictionary data using LIFETIME](#refreshing-dictionary-data-using-lifetime).
- `secure` - Use ssl for connection.
- `query` – The custom query. Optional parameter.
@@ -1849,7 +1849,7 @@ Setting fields:
- `db` – Name of the database.
- `table` – Name of the table.
- `where` – The selection criteria. The syntax for conditions is the same as for `WHERE` clause in PostgreSQL. For example, `id > 10 AND id < 20`. Optional parameter.
-- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Updating dictionaries](#dictionary-updates).
+- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Refreshing dictionary data using LIFETIME](#refreshing-dictionary-data-using-lifetime).
- `query` – The custom query. Optional parameter.
:::note
@@ -1873,7 +1873,7 @@ LAYOUT(FLAT())
LIFETIME(0);
```
-## Dictionary Key and Fields {#dictionary-key-and-fields}
+## Dictionary Key and Fields
@@ -1963,7 +1963,7 @@ PRIMARY KEY Id
### Composite Key
-The key can be a `tuple` from any types of fields. The [layout](#storig-dictionaries-in-memory) in this case must be `complex_key_hashed` or `complex_key_cache`.
+The key can be a `tuple` from any types of fields. The [layout](#storing-dictionaries-in-memory) in this case must be `complex_key_hashed` or `complex_key_cache`.
:::tip
A composite key can consist of a single element. This makes it possible to use a string as the key, for instance.
@@ -2030,17 +2030,17 @@ CREATE DICTIONARY somename (
Configuration fields:
-| Tag | Description | Required |
-|------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|
-| `name` | Column name. | Yes |
-| `type` | ClickHouse data type: [UInt8](../../sql-reference/data-types/int-uint.md), [UInt16](../../sql-reference/data-types/int-uint.md), [UInt32](../../sql-reference/data-types/int-uint.md), [UInt64](../../sql-reference/data-types/int-uint.md), [Int8](../../sql-reference/data-types/int-uint.md), [Int16](../../sql-reference/data-types/int-uint.md), [Int32](../../sql-reference/data-types/int-uint.md), [Int64](../../sql-reference/data-types/int-uint.md), [Float32](../../sql-reference/data-types/float.md), [Float64](../../sql-reference/data-types/float.md), [UUID](../../sql-reference/data-types/uuid.md), [Decimal32](../../sql-reference/data-types/decimal.md), [Decimal64](../../sql-reference/data-types/decimal.md), [Decimal128](../../sql-reference/data-types/decimal.md), [Decimal256](../../sql-reference/data-types/decimal.md),[Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md), [DateTime64](../../sql-reference/data-types/datetime64.md), [String](../../sql-reference/data-types/string.md), [Array](../../sql-reference/data-types/array.md).
ClickHouse tries to cast value from dictionary to the specified data type. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the MySQL source table, but it can be uploaded as `String` in ClickHouse.
[Nullable](../../sql-reference/data-types/nullable.md) is currently supported for [Flat](#flat), [Hashed](#hashed), [ComplexKeyHashed](#complex_key_hashed), [Direct](#direct), [ComplexKeyDirect](#complex_key_direct), [RangeHashed](#range_hashed), Polygon, [Cache](#cache), [ComplexKeyCache](#complex_key_cache), [SSDCache](#ssd_cache), [SSDComplexKeyCache](#complex_key_ssd_cache) dictionaries. In [IPTrie](#ip_trie) dictionaries `Nullable` types are not supported. | Yes |
-| `null_value` | Default value for a non-existing element.
In the example, it is an empty string. [NULL](../syntax.md#null) value can be used only for the `Nullable` types (see the previous line with types description). | Yes |
-| `expression` | [Expression](../../sql-reference/syntax.md#expressions) that ClickHouse executes on the value.
The expression can be a column name in the remote SQL database. Thus, you can use it to create an alias for the remote column.
Default value: no expression. | No |
-| `hierarchical` | If `true`, the attribute contains the value of a parent key for the current key. See [Hierarchical Dictionaries](#hierarchical-dictionaries).
Default value: `false`. | No |
-| `injective` | Flag that shows whether the `id -> attribute` image is [injective](https://en.wikipedia.org/wiki/Injective_function).
If `true`, ClickHouse can automatically place after the `GROUP BY` clause the requests to dictionaries with injection. Usually it significantly reduces the amount of such requests.
Default value: `false`. | No |
-| `is_object_id` | Flag that shows whether the query is executed for a MongoDB document by `ObjectID`.
Default value: `false`.
+| Tag | Description | Required |
+|------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|
+| `name` | Column name. | Yes |
+| `type` | ClickHouse data type: [UInt8](../../sql-reference/data-types/int-uint.md), [UInt16](../../sql-reference/data-types/int-uint.md), [UInt32](../../sql-reference/data-types/int-uint.md), [UInt64](../../sql-reference/data-types/int-uint.md), [Int8](../../sql-reference/data-types/int-uint.md), [Int16](../../sql-reference/data-types/int-uint.md), [Int32](../../sql-reference/data-types/int-uint.md), [Int64](../../sql-reference/data-types/int-uint.md), [Float32](../../sql-reference/data-types/float.md), [Float64](../../sql-reference/data-types/float.md), [UUID](../../sql-reference/data-types/uuid.md), [Decimal32](../../sql-reference/data-types/decimal.md), [Decimal64](../../sql-reference/data-types/decimal.md), [Decimal128](../../sql-reference/data-types/decimal.md), [Decimal256](../../sql-reference/data-types/decimal.md),[Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md), [DateTime64](../../sql-reference/data-types/datetime64.md), [String](../../sql-reference/data-types/string.md), [Array](../../sql-reference/data-types/array.md).
ClickHouse tries to cast value from dictionary to the specified data type. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the MySQL source table, but it can be uploaded as `String` in ClickHouse.
[Nullable](../../sql-reference/data-types/nullable.md) is currently supported for [Flat](#flat), [Hashed](#hashed), [ComplexKeyHashed](#complex_key_hashed), [Direct](#direct), [ComplexKeyDirect](#complex_key_direct), [RangeHashed](#range_hashed), Polygon, [Cache](#cache), [ComplexKeyCache](#complex_key_cache), [SSDCache](#ssd_cache), [SSDComplexKeyCache](#complex_key_ssd_cache) dictionaries. In [IPTrie](#ip_trie) dictionaries `Nullable` types are not supported. | Yes |
+| `null_value` | Default value for a non-existing element.
In the example, it is an empty string. [NULL](../syntax.md#null) value can be used only for the `Nullable` types (see the previous line with types description). | Yes |
+| `expression` | [Expression](../../sql-reference/syntax.md#expressions) that ClickHouse executes on the value.
The expression can be a column name in the remote SQL database. Thus, you can use it to create an alias for the remote column.
Default value: no expression. | No |
+| `hierarchical` | If `true`, the attribute contains the value of a parent key for the current key. See [Hierarchical Dictionaries](#hierarchical-dictionaries).
Default value: `false`. | No |
+| `injective` | Flag that shows whether the `id -> attribute` image is [injective](https://en.wikipedia.org/wiki/Injective_function).
If `true`, ClickHouse can automatically place after the `GROUP BY` clause the requests to dictionaries with injection. Usually it significantly reduces the amount of such requests.
Default value: `false`. | No |
+| `is_object_id` | Flag that shows whether the query is executed for a MongoDB document by `ObjectID`.
Default value: `false`.
-## Hierarchical Dictionaries {#hierarchical-dictionaries}
+## Hierarchical Dictionaries
ClickHouse supports hierarchical dictionaries with a [numeric key](#numeric-key).
@@ -2165,7 +2165,7 @@ Points can be specified as an array or a tuple of their coordinates. In the curr
The user can upload their own data in all formats supported by ClickHouse.
-There are 3 types of [in-memory storage](#storig-dictionaries-in-memory) available:
+There are 3 types of [in-memory storage](#storing-dictionaries-in-memory) available:
- `POLYGON_SIMPLE`. This is a naive implementation, where a linear pass through all polygons is made for each query, and membership is checked for each one without using additional indexes.
@@ -2435,7 +2435,7 @@ LIFETIME(0)
LAYOUT(regexp_tree);
```
-## Embedded Dictionaries {#embedded-dictionaries}
+## Embedded Dictionaries
diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md
index 7b52fbff714..d87ca4a0fe7 100644
--- a/docs/en/sql-reference/functions/array-functions.md
+++ b/docs/en/sql-reference/functions/array-functions.md
@@ -1261,7 +1261,7 @@ SELECT arraySort((x) -> -x, [1, 2, 3]) as res;
└─────────┘
```
-For each element of the source array, the lambda function returns the sorting key, that is, \[1 –\> -1, 2 –\> -2, 3 –\> -3\]. Since the `arraySort` function sorts the keys in ascending order, the result is \[3, 2, 1\]. Thus, the `(x) –> -x` lambda function sets the [descending order](#reverse-sort) in a sorting.
+For each element of the source array, the lambda function returns the sorting key, that is, \[1 –\> -1, 2 –\> -2, 3 –\> -3\]. Since the `arraySort` function sorts the keys in ascending order, the result is \[3, 2, 1\]. Thus, the `(x) –> -x` lambda function sets the [descending order](#arrayreversesort) in a sorting.
The lambda function can accept multiple arguments. In this case, you need to pass the `arraySort` function several arrays of identical length that the arguments of lambda function will correspond to. The resulting array will consist of elements from the first input array; elements from the next input array(s) specify the sorting keys. For example:
@@ -1307,10 +1307,15 @@ To improve sorting efficiency, the [Schwartzian transform](https://en.wikipedia.
Same as `arraySort` with additional `limit` argument allowing partial sorting. Returns an array of the same size as the original array where elements in range `[1..limit]` are sorted in ascending order. Remaining elements `(limit..N]` shall contain elements in unspecified order.
-## arrayReverseSort(\[func,\] arr, ...) {#reverse-sort}
+## arrayReverseSort
Sorts the elements of the `arr` array in descending order. If the `func` function is specified, `arr` is sorted according to the result of the `func` function applied to the elements of the array, and then the sorted array is reversed. If `func` accepts multiple arguments, the `arrayReverseSort` function is passed several arrays that the arguments of `func` will correspond to. Detailed examples are shown at the end of `arrayReverseSort` description.
+**Syntax**
+
+```sql
+arrayReverseSort([func,] arr, ...)
+```
Example of integer values sorting:
``` sql
@@ -1907,10 +1912,16 @@ FROM numbers(1,10);
- [arrayReduce](#arrayreduce)
-## arrayReverse(arr)
+## arrayReverse
Returns an array of the same size as the original array containing the elements in reverse order.
+**Syntax**
+
+```sql
+arrayReverse(arr)
+```
+
Example:
``` sql
diff --git a/docs/en/sql-reference/functions/bit-functions.md b/docs/en/sql-reference/functions/bit-functions.md
index a48893b93bf..5ab7e07fcad 100644
--- a/docs/en/sql-reference/functions/bit-functions.md
+++ b/docs/en/sql-reference/functions/bit-functions.md
@@ -173,7 +173,7 @@ See function [substring](string-functions.md#substring).
## bitTest
-Takes any integer and converts it into [binary form](https://en.wikipedia.org/wiki/Binary_number), returns the value of a bit at specified position. The countdown starts from 0 from the right to the left.
+Takes any integer and converts it into [binary form](https://en.wikipedia.org/wiki/Binary_number), returns the value of a bit at specified position. Counting is right-to-left, starting at 0.
**Syntax**
@@ -226,7 +226,7 @@ Result:
## bitTestAll
-Returns result of [logical conjuction](https://en.wikipedia.org/wiki/Logical_conjunction) (AND operator) of all bits at given positions. The countdown starts from 0 from the right to the left.
+Returns result of [logical conjuction](https://en.wikipedia.org/wiki/Logical_conjunction) (AND operator) of all bits at given positions. Counting is right-to-left, starting at 0.
The conjuction for bit-wise operations:
@@ -289,7 +289,7 @@ Result:
## bitTestAny
-Returns result of [logical disjunction](https://en.wikipedia.org/wiki/Logical_disjunction) (OR operator) of all bits at given positions. The countdown starts from 0 from the right to the left.
+Returns result of [logical disjunction](https://en.wikipedia.org/wiki/Logical_disjunction) (OR operator) of all bits at given positions. Counting is right-to-left, starting at 0.
The disjunction for bit-wise operations:
diff --git a/docs/en/sql-reference/functions/bitmap-functions.md b/docs/en/sql-reference/functions/bitmap-functions.md
index a5c8a663b71..d30c0f4dde4 100644
--- a/docs/en/sql-reference/functions/bitmap-functions.md
+++ b/docs/en/sql-reference/functions/bitmap-functions.md
@@ -74,7 +74,7 @@ bitmapSubsetInRange(bitmap, range_start, range_end)
**Arguments**
-- `bitmap` – [Bitmap object](#bitmap_functions-bitmapbuild).
+- `bitmap` – [Bitmap object](#bitmapbuild).
- `range_start` – Start of the range (inclusive). [UInt32](../data-types/int-uint.md).
- `range_end` – End of the range (exclusive). [UInt32](../data-types/int-uint.md).
@@ -104,7 +104,7 @@ bitmapSubsetLimit(bitmap, range_start, cardinality_limit)
**Arguments**
-- `bitmap` – [Bitmap object](#bitmap_functions-bitmapbuild).
+- `bitmap` – [Bitmap object](#bitmapbuild).
- `range_start` – Start of the range (inclusive). [UInt32](../data-types/int-uint.md).
- `cardinality_limit` – Maximum cardinality of the subset. [UInt32](../data-types/int-uint.md).
@@ -134,7 +134,7 @@ subBitmap(bitmap, offset, cardinality_limit)
**Arguments**
-- `bitmap` – The bitmap. [Bitmap object](#bitmap_functions-bitmapbuild).
+- `bitmap` – The bitmap. [Bitmap object](#bitmapbuild).
- `offset` – The position of the first element of the subset. [UInt32](../data-types/int-uint.md).
- `cardinality_limit` – The maximum number of elements in the subset. [UInt32](../data-types/int-uint.md).
@@ -162,7 +162,7 @@ bitmapContains(bitmap, needle)
**Arguments**
-- `bitmap` – [Bitmap object](#bitmap_functions-bitmapbuild).
+- `bitmap` – [Bitmap object](#bitmapbuild).
- `needle` – Searched bit value. [UInt32](../data-types/int-uint.md).
**Returned values**
@@ -188,7 +188,7 @@ Result:
Checks whether two bitmaps intersect.
-If `bitmap2` contains exactly one element, consider using [bitmapContains](#bitmap_functions-bitmapcontains) instead as it works more efficiently.
+If `bitmap2` contains exactly one element, consider using [bitmapContains](#bitmapcontains) instead as it works more efficiently.
**Syntax**
diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md
index 4092c83954a..b532e0de8f0 100644
--- a/docs/en/sql-reference/functions/date-time-functions.md
+++ b/docs/en/sql-reference/functions/date-time-functions.md
@@ -83,7 +83,7 @@ Result:
```
## makeDate32
-Like [makeDate](#makeDate) but produces a [Date32](../data-types/date32.md).
+Like [makeDate](#makedate) but produces a [Date32](../data-types/date32.md).
## makeDateTime
@@ -214,7 +214,7 @@ Result:
**See also**
-- [serverTimeZone](#serverTimeZone)
+- [serverTimeZone](#servertimezone)
## serverTimeZone
@@ -249,7 +249,7 @@ Result:
**See also**
-- [timeZone](#timeZone)
+- [timeZone](#timezone)
## toTimeZone
@@ -305,7 +305,7 @@ int32samoa: 1546300800
**See Also**
-- [formatDateTime](#formatDateTime) - supports non-constant timezone.
+- [formatDateTime](#formatdatetime) - supports non-constant timezone.
- [toString](type-conversion-functions.md#tostring) - supports non-constant timezone.
## timeZoneOf
@@ -1006,7 +1006,7 @@ toStartOfWeek(t[, mode[, timezone]])
**Arguments**
- `t` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
-- `mode` - determines the first day of the week as described in the [toWeek()](date-time-functions#toweek) function
+- `mode` - determines the first day of the week as described in the [toWeek()](#toweek) function
- `timezone` - Optional parameter, it behaves like any other conversion function
**Returned value**
@@ -1049,7 +1049,7 @@ toLastDayOfWeek(t[, mode[, timezone]])
**Arguments**
- `t` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
-- `mode` - determines the last day of the week as described in the [toWeek()](date-time-functions#toweek) function
+- `mode` - determines the last day of the week as described in the [toWeek](#toweek) function
- `timezone` - Optional parameter, it behaves like any other conversion function
**Returned value**
@@ -1719,7 +1719,7 @@ Result:
**See Also**
-- [fromDaysSinceYearZero](#fromDaysSinceYearZero)
+- [fromDaysSinceYearZero](#fromdayssinceyearzero)
## fromDaysSinceYearZero
@@ -1759,11 +1759,11 @@ Result:
**See Also**
-- [toDaysSinceYearZero](#toDaysSinceYearZero)
+- [toDaysSinceYearZero](#todayssinceyearzero)
## fromDaysSinceYearZero32
-Like [fromDaysSinceYearZero](#fromDaysSinceYearZero) but returns a [Date32](../data-types/date32.md).
+Like [fromDaysSinceYearZero](#fromdayssinceyearzero) but returns a [Date32](../data-types/date32.md).
## age
@@ -1982,7 +1982,7 @@ Result:
**See Also**
-- [toStartOfInterval](#tostartofintervaldate_or_date_with_time-interval-x-unit--time_zone)
+- [toStartOfInterval](#tostartofinterval)
## date\_add
@@ -2055,7 +2055,7 @@ Result:
**See Also**
-- [addDate](#addDate)
+- [addDate](#adddate)
## date\_sub
@@ -2129,7 +2129,7 @@ Result:
**See Also**
-- [subDate](#subDate)
+- [subDate](#subdate)
## timestamp\_add
@@ -2310,7 +2310,7 @@ Alias: `SUBDATE`
- [date_sub](#date_sub)
-## now {#now}
+## now
Returns the current date and time at the moment of query analysis. The function is a constant expression.
@@ -3609,7 +3609,7 @@ SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64
└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
-## formatDateTime {#formatDateTime}
+## formatDateTime
Formats a Time according to the given Format string. Format is a constant expression, so you cannot have multiple formats for a single result column.
@@ -3734,10 +3734,9 @@ LIMIT 10
**See Also**
-- [formatDateTimeInJodaSyntax](##formatDateTimeInJodaSyntax)
+- [formatDateTimeInJodaSyntax](#formatdatetimeinjodasyntax)
-
-## formatDateTimeInJodaSyntax {#formatDateTimeInJodaSyntax}
+## formatDateTimeInJodaSyntax
Similar to formatDateTime, except that it formats datetime in Joda style instead of MySQL style. Refer to https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html.
@@ -3902,11 +3901,11 @@ Result:
**See Also**
-- [fromUnixTimestampInJodaSyntax](##fromUnixTimestampInJodaSyntax)
+- [fromUnixTimestampInJodaSyntax](#fromunixtimestampinjodasyntax)
## fromUnixTimestampInJodaSyntax
-Same as [fromUnixTimestamp](#fromUnixTimestamp) but when called in the second way (two or three arguments), the formatting is performed using [Joda style](https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html) instead of MySQL style.
+Same as [fromUnixTimestamp](#fromunixtimestamp) but when called in the second way (two or three arguments), the formatting is performed using [Joda style](https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html) instead of MySQL style.
**Example:**
@@ -4121,7 +4120,7 @@ Result:
Returns the current date and time at the moment of query analysis. The function is a constant expression.
:::note
-This function gives the same result that `now('UTC')` would. It was added only for MySQL support and [`now`](#now-now) is the preferred usage.
+This function gives the same result that `now('UTC')` would. It was added only for MySQL support and [`now`](#now) is the preferred usage.
:::
**Syntax**
diff --git a/docs/en/sql-reference/functions/ext-dict-functions.md b/docs/en/sql-reference/functions/ext-dict-functions.md
index 82c21ce40c8..093ee690d47 100644
--- a/docs/en/sql-reference/functions/ext-dict-functions.md
+++ b/docs/en/sql-reference/functions/ext-dict-functions.md
@@ -12,7 +12,7 @@ For dictionaries created with [DDL queries](../../sql-reference/statements/creat
For information on connecting and configuring dictionaries, see [Dictionaries](../../sql-reference/dictionaries/index.md).
-## dictGet, dictGetOrDefault, dictGetOrNull {#dictGet}
+## dictGet, dictGetOrDefault, dictGetOrNull
Retrieves values from a dictionary.
diff --git a/docs/en/sql-reference/functions/geo/geohash.md b/docs/en/sql-reference/functions/geo/geohash.md
index 8abc8006e5d..b6ac7a74092 100644
--- a/docs/en/sql-reference/functions/geo/geohash.md
+++ b/docs/en/sql-reference/functions/geo/geohash.md
@@ -4,6 +4,8 @@ sidebar_label: Geohash
title: "Functions for Working with Geohash"
---
+## Geohash
+
[Geohash](https://en.wikipedia.org/wiki/Geohash) is the geocode system, which subdivides Earth’s surface into buckets of grid shape and encodes each cell into a short string of letters and digits. It is a hierarchical data structure, so the longer is the geohash string, the more precise is the geographic location.
If you need to manually convert geographic coordinates to geohash strings, you can use [geohash.org](http://geohash.org/).
diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md
index bcdd457964a..5fbc2adf2fa 100644
--- a/docs/en/sql-reference/functions/geo/h3.md
+++ b/docs/en/sql-reference/functions/geo/h3.md
@@ -4,6 +4,8 @@ sidebar_label: H3 Indexes
title: "Functions for Working with H3 Indexes"
---
+## H3 Index
+
[H3](https://eng.uber.com/h3/) is a geographical indexing system where Earth’s surface divided into a grid of even hexagonal cells. This system is hierarchical, i. e. each hexagon on the top level ("parent") can be split into seven even but smaller ones ("children"), and so on.
The level of the hierarchy is called `resolution` and can receive a value from `0` till `15`, where `0` is the `base` level with the largest and coarsest cells.
@@ -16,7 +18,7 @@ The full description of the H3 system is available at [the Uber Engineering site
## h3IsValid
-Verifies whether the number is a valid [H3](#h3index) index.
+Verifies whether the number is a valid [H3](#h3-index) index.
**Syntax**
@@ -51,7 +53,7 @@ Result:
## h3GetResolution
-Defines the resolution of the given [H3](#h3index) index.
+Defines the resolution of the given [H3](#h3-index) index.
**Syntax**
@@ -86,7 +88,7 @@ Result:
## h3EdgeAngle
-Calculates the average length of the [H3](#h3index) hexagon edge in grades.
+Calculates the average length of the [H3](#h3-index) hexagon edge in grades.
**Syntax**
@@ -100,7 +102,7 @@ h3EdgeAngle(resolution)
**Returned values**
-- The average length of the [H3](#h3index) hexagon edge in grades. [Float64](../../data-types/float.md).
+- The average length of the [H3](#h3-index) hexagon edge in grades. [Float64](../../data-types/float.md).
**Example**
@@ -120,7 +122,7 @@ Result:
## h3EdgeLengthM
-Calculates the average length of the [H3](#h3index) hexagon edge in meters.
+Calculates the average length of the [H3](#h3-index) hexagon edge in meters.
**Syntax**
@@ -134,7 +136,7 @@ h3EdgeLengthM(resolution)
**Returned values**
-- The average length of the [H3](#h3index) hexagon edge in meters. [Float64](../../data-types/float.md).
+- The average length of the [H3](#h3-index) hexagon edge in meters. [Float64](../../data-types/float.md).
**Example**
@@ -154,7 +156,7 @@ Result:
## h3EdgeLengthKm
-Calculates the average length of the [H3](#h3index) hexagon edge in kilometers.
+Calculates the average length of the [H3](#h3-index) hexagon edge in kilometers.
**Syntax**
@@ -168,7 +170,7 @@ h3EdgeLengthKm(resolution)
**Returned values**
-- The average length of the [H3](#h3index) hexagon edge in kilometers. [Float64](../../data-types/float.md).
+- The average length of the [H3](#h3-index) hexagon edge in kilometers. [Float64](../../data-types/float.md).
**Example**
@@ -188,7 +190,7 @@ Result:
## geoToH3
-Returns [H3](#h3index) point index `(lon, lat)` with specified resolution.
+Returns [H3](#h3-index) point index `(lon, lat)` with specified resolution.
**Syntax**
@@ -225,7 +227,7 @@ Result:
## h3ToGeo
-Returns the centroid longitude and latitude corresponding to the provided [H3](#h3index) index.
+Returns the centroid longitude and latitude corresponding to the provided [H3](#h3-index) index.
**Syntax**
@@ -294,7 +296,7 @@ Result:
## h3kRing
- Lists all the [H3](#h3index) hexagons in the raduis of `k` from the given hexagon in random order.
+ Lists all the [H3](#h3-index) hexagons in the raduis of `k` from the given hexagon in random order.
**Syntax**
@@ -335,7 +337,7 @@ Result:
## h3GetBaseCell
-Returns the base cell number of the [H3](#h3index) index.
+Returns the base cell number of the [H3](#h3-index) index.
**Syntax**
@@ -437,7 +439,7 @@ Result:
## h3IndexesAreNeighbors
-Returns whether or not the provided [H3](#h3index) indexes are neighbors.
+Returns whether or not the provided [H3](#h3-index) indexes are neighbors.
**Syntax**
@@ -473,7 +475,7 @@ Result:
## h3ToChildren
-Returns an array of child indexes for the given [H3](#h3index) index.
+Returns an array of child indexes for the given [H3](#h3-index) index.
**Syntax**
@@ -508,7 +510,7 @@ Result:
## h3ToParent
-Returns the parent (coarser) index containing the given [H3](#h3index) index.
+Returns the parent (coarser) index containing the given [H3](#h3-index) index.
**Syntax**
@@ -609,7 +611,7 @@ Result:
## h3GetResolution
-Returns the resolution of the [H3](#h3index) index.
+Returns the resolution of the [H3](#h3-index) index.
**Syntax**
@@ -643,7 +645,7 @@ Result:
## h3IsResClassIII
-Returns whether [H3](#h3index) index has a resolution with Class III orientation.
+Returns whether [H3](#h3-index) index has a resolution with Class III orientation.
**Syntax**
@@ -678,7 +680,7 @@ Result:
## h3IsPentagon
-Returns whether this [H3](#h3index) index represents a pentagonal cell.
+Returns whether this [H3](#h3-index) index represents a pentagonal cell.
**Syntax**
@@ -713,7 +715,7 @@ Result:
## h3GetFaces
-Returns icosahedron faces intersected by a given [H3](#h3index) index.
+Returns icosahedron faces intersected by a given [H3](#h3-index) index.
**Syntax**
@@ -815,7 +817,7 @@ Result:
## h3ToCenterChild
-Returns the center child (finer) [H3](#h3index) index contained by given [H3](#h3index) at the given resolution.
+Returns the center child (finer) [H3](#h3-index) index contained by given [H3](#h3-index) at the given resolution.
**Syntax**
@@ -830,7 +832,7 @@ h3ToCenterChild(index, resolution)
**Returned values**
-- [H3](#h3index) index of the center child contained by given [H3](#h3index) at the given resolution. [UInt64](../../data-types/int-uint.md).
+- [H3](#h3-index) index of the center child contained by given [H3](#h3-index) at the given resolution. [UInt64](../../data-types/int-uint.md).
**Example**
diff --git a/docs/en/sql-reference/functions/geo/polygon.md b/docs/en/sql-reference/functions/geo/polygon.md
index c2572779ada..25a7a1fac8e 100644
--- a/docs/en/sql-reference/functions/geo/polygon.md
+++ b/docs/en/sql-reference/functions/geo/polygon.md
@@ -142,6 +142,34 @@ SELECT readWKTPoint('POINT (1.2 3.4)');
(1.2,3.4)
```
+## readWKTLineString
+
+Parses a Well-Known Text (WKT) representation of a LineString geometry and returns it in the internal ClickHouse format.
+
+### Syntax
+
+```sql
+readWKTLineString(wkt_string)
+```
+
+### Arguments
+
+- `wkt_string`: The input WKT string representing a LineString geometry.
+
+### Returned value
+
+The function returns a ClickHouse internal representation of the linestring geometry.
+
+### Example
+
+```sql
+SELECT readWKTLineString('LINESTRING (1 1, 2 2, 3 3, 1 1)');
+```
+
+```response
+[(1,1),(2,2),(3,3),(1,1)]
+```
+
## readWKTRing
Parses a Well-Known Text (WKT) representation of a Polygon geometry and returns a ring (closed linestring) in the internal ClickHouse format.
@@ -163,7 +191,7 @@ The function returns a ClickHouse internal representation of the ring (closed li
### Example
```sql
-SELECT readWKTRing('LINESTRING (1 1, 2 2, 3 3, 1 1)');
+SELECT readWKTRing('POLYGON ((1 1, 2 2, 3 3, 1 1))');
```
```response
diff --git a/docs/en/sql-reference/functions/geo/s2.md b/docs/en/sql-reference/functions/geo/s2.md
index 3165b21318b..e022ce870b0 100644
--- a/docs/en/sql-reference/functions/geo/s2.md
+++ b/docs/en/sql-reference/functions/geo/s2.md
@@ -5,6 +5,8 @@ sidebar_label: S2 Geometry
# Functions for Working with S2 Index
+## S2Index
+
[S2](https://s2geometry.io/) is a geographical indexing system where all geographical data is represented on a three-dimensional sphere (similar to a globe).
In the S2 library points are represented as the S2 Index - a specific number which encodes internally a point on the surface of a unit sphere, unlike traditional (latitude, longitude) pairs. To get the S2 point index for a given point specified in the format (latitude, longitude) use the [geoToS2](#geotos2) function. Also, you can use the [s2ToGeo](#s2togeo) function for getting geographical coordinates corresponding to the specified S2 point index.
diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md
index 506114038f7..e431ed75465 100644
--- a/docs/en/sql-reference/functions/hash-functions.md
+++ b/docs/en/sql-reference/functions/hash-functions.md
@@ -45,13 +45,13 @@ SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')
Calculates the MD4 from a string and returns the resulting set of bytes as FixedString(16).
-## MD5 {#md5}
+## MD5
Calculates the MD5 from a string and returns the resulting set of bytes as FixedString(16).
If you do not need MD5 in particular, but you need a decent cryptographic 128-bit hash, use the ‘sipHash128’ function instead.
If you want to get the same result as output by the md5sum utility, use lower(hex(MD5(s))).
-## sipHash64 {#siphash64}
+## sipHash64
Produces a 64-bit [SipHash](https://en.wikipedia.org/wiki/SipHash) hash value.
diff --git a/docs/en/sql-reference/functions/ip-address-functions.md b/docs/en/sql-reference/functions/ip-address-functions.md
index 5b6a3aef2c8..11a7749b33d 100644
--- a/docs/en/sql-reference/functions/ip-address-functions.md
+++ b/docs/en/sql-reference/functions/ip-address-functions.md
@@ -295,7 +295,7 @@ Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns null
## toIPv6
Converts a string form of IPv6 address to [IPv6](../data-types/ipv6.md) type. If the IPv6 address has an invalid format, returns an empty value.
-Similar to [IPv6StringToNum](#ipv6stringtonums) function, which converts IPv6 address to binary format.
+Similar to [IPv6StringToNum](#ipv6stringtonum) function, which converts IPv6 address to binary format.
If the input string contains a valid IPv4 address, then the IPv6 equivalent of the IPv4 address is returned.
diff --git a/docs/en/sql-reference/functions/json-functions.md b/docs/en/sql-reference/functions/json-functions.md
index 5d73c9a83b3..7bff6a6cba5 100644
--- a/docs/en/sql-reference/functions/json-functions.md
+++ b/docs/en/sql-reference/functions/json-functions.md
@@ -5,10 +5,10 @@ sidebar_label: JSON
---
There are two sets of functions to parse JSON:
- - [`simpleJSON*` (`visitParam*`)](#simplejson--visitparam-functions) which is made for parsing a limited subset of JSON extremely fast.
+ - [`simpleJSON*` (`visitParam*`)](#simplejson-visitparam-functions) which is made for parsing a limited subset of JSON extremely fast.
- [`JSONExtract*`](#jsonextract-functions) which is made for parsing ordinary JSON.
-## simpleJSON / visitParam functions
+## simpleJSON (visitParam) functions
ClickHouse has special functions for working with simplified JSON. All these JSON functions are based on strong assumptions about what the JSON can be. They try to do as little as possible to get the job done as quickly as possible.
diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md
index 12098efc635..b9b5c6d7a05 100644
--- a/docs/en/sql-reference/functions/math-functions.md
+++ b/docs/en/sql-reference/functions/math-functions.md
@@ -8,7 +8,7 @@ sidebar_label: Mathematical
## e
-Returns e ([Euler's constant](https://en.wikipedia.org/wiki/Euler%27s_constant)).
+Returns $e$ ([Euler's constant](https://en.wikipedia.org/wiki/Euler%27s_constant)).
**Syntax**
@@ -22,7 +22,7 @@ Type: [Float64](../data-types/float.md).
## pi
-Returns π ([Pi](https://en.wikipedia.org/wiki/Pi)).
+Returns $\pi$ ([Pi](https://en.wikipedia.org/wiki/Pi)).
**Syntax**
@@ -35,7 +35,7 @@ Type: [Float64](../data-types/float.md).
## exp
-Returns e to the power of the given argument.
+Returns $e^{x}$, where x is the given argument to the function.
**Syntax**
@@ -47,6 +47,22 @@ exp(x)
- `x` - [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
+**Example**
+
+Query:
+
+```sql
+SELECT round(exp(-1), 4);
+```
+
+Result:
+
+```response
+┌─round(exp(-1), 4)─┐
+│ 0.3679 │
+└───────────────────┘
+```
+
**Returned value**
Type: [Float*](../data-types/float.md).
@@ -91,7 +107,7 @@ Type: [Float*](../data-types/float.md).
## intExp2
-Like `exp` but returns a UInt64.
+Like [`exp`](#exp) but returns a UInt64.
**Syntax**
@@ -137,7 +153,7 @@ Type: [Float*](../data-types/float.md).
## intExp10
-Like `exp10` but returns a UInt64.
+Like [`exp10`](#exp10) but returns a UInt64.
**Syntax**
@@ -197,7 +213,7 @@ Type: [Float*](../data-types/float.md).
## erf
-If `x` is non-negative, then `erf(x / σ√2)` is the probability that a random variable having a normal distribution with standard deviation `σ` takes the value that is separated from the expected value by more than `x`.
+If `x` is non-negative, then $erf(\frac{x}{\sigma\sqrt{2}})$ is the probability that a random variable having a normal distribution with standard deviation $\sigma$ takes the value that is separated from the expected value by more than `x`.
**Syntax**
@@ -229,7 +245,7 @@ SELECT erf(3 / sqrt(2));
## erfc
-Returns a number close to `1 - erf(x)` without loss of precision for large ‘x’ values.
+Returns a number close to $1-erf(x)$ without loss of precision for large `x` values.
**Syntax**
@@ -403,7 +419,7 @@ Type: [Float*](../data-types/float.md).
## pow
-Returns `x` to the power of `y`.
+Returns $x^y$.
**Syntax**
@@ -434,11 +450,11 @@ cosh(x)
**Arguments**
-- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
+- `x` — The angle, in radians. Values from the interval: $-\infty \lt x \lt +\infty$. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
**Returned value**
-- Values from the interval: `1 <= cosh(x) < +∞`.
+- Values from the interval: $1 \le cosh(x) \lt +\infty$.
Type: [Float64](../data-types/float.md#float32-float64).
@@ -468,11 +484,11 @@ acosh(x)
**Arguments**
-- `x` — Hyperbolic cosine of angle. Values from the interval: `1 <= x < +∞`. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
+- `x` — Hyperbolic cosine of angle. Values from the interval: $1 \le x \lt +\infty$. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
**Returned value**
-- The angle, in radians. Values from the interval: `0 <= acosh(x) < +∞`.
+- The angle, in radians. Values from the interval: $0 \le acosh(x) \lt +\infty$.
Type: [Float64](../data-types/float.md#float32-float64).
@@ -502,11 +518,11 @@ sinh(x)
**Arguments**
-- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
+- `x` — The angle, in radians. Values from the interval: $-\infty \lt x \lt +\infty$. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
**Returned value**
-- Values from the interval: `-∞ < sinh(x) < +∞`.
+- Values from the interval: $-\infty \lt sinh(x) \lt +\infty$.
Type: [Float64](../data-types/float.md#float32-float64).
@@ -536,11 +552,11 @@ asinh(x)
**Arguments**
-- `x` — Hyperbolic sine of angle. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
+- `x` — Hyperbolic sine of angle. Values from the interval: $-\infty \lt x \lt +\infty$. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
**Returned value**
-- The angle, in radians. Values from the interval: `-∞ < asinh(x) < +∞`.
+- The angle, in radians. Values from the interval: $-\infty \lt asinh(x) \lt +\infty$.
Type: [Float64](../data-types/float.md#float32-float64).
@@ -569,11 +585,11 @@ tanh(x)
**Arguments**
-- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
+- `x` — The angle, in radians. Values from the interval: $-\infty \lt x \lt +\infty$. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
**Returned value**
-- Values from the interval: `-1 < tanh(x) < 1`.
+- Values from the interval: $-1 \lt tanh(x) \lt 1$.
Type: [Float*](../data-types/float.md#float32-float64).
@@ -601,11 +617,11 @@ atanh(x)
**Arguments**
-- `x` — Hyperbolic tangent of angle. Values from the interval: `–1 < x < 1`. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
+- `x` — Hyperbolic tangent of angle. Values from the interval: $-1 \lt x \lt 1$. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
**Returned value**
-- The angle, in radians. Values from the interval: `-∞ < atanh(x) < +∞`.
+- The angle, in radians. Values from the interval: $-\infty \lt atanh(x) \lt +\infty$.
Type: [Float64](../data-types/float.md#float32-float64).
@@ -640,7 +656,7 @@ atan2(y, x)
**Returned value**
-- The angle `θ` such that `−π < θ ≤ π`, in radians.
+- The angle `θ` such that $-\pi \lt 0 \le \pi$, in radians.
Type: [Float64](../data-types/float.md#float32-float64).
@@ -705,11 +721,11 @@ log1p(x)
**Arguments**
-- `x` — Values from the interval: `-1 < x < +∞`. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
+- `x` — Values from the interval: $-1 \lt x \lt +\infty$. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
**Returned value**
-- Values from the interval: `-∞ < log1p(x) < +∞`.
+- Values from the interval: $-\infty < log1p(x) \lt +\infty$.
Type: [Float64](../data-types/float.md#float32-float64).
@@ -739,7 +755,7 @@ sign(x)
**Arguments**
-- `x` — Values from `-∞` to `+∞`. Support all numeric types in ClickHouse.
+- `x` — Values from $-\infty$ to $+\infty$. Supports all numeric types in ClickHouse.
**Returned value**
@@ -804,7 +820,7 @@ sigmoid(x)
**Parameters**
-- `x` — input value. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
+- `x` — input value. Values from the interval: $-\infty \lt x \lt +\infty$. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md) or [Decimal*](../data-types/decimal.md).
**Returned value**
diff --git a/docs/en/sql-reference/functions/nlp-functions.md b/docs/en/sql-reference/functions/nlp-functions.md
index 4bfa181a35f..7057ebebfe4 100644
--- a/docs/en/sql-reference/functions/nlp-functions.md
+++ b/docs/en/sql-reference/functions/nlp-functions.md
@@ -6,26 +6,297 @@ sidebar_label: NLP (experimental)
# Natural Language Processing (NLP) Functions
-:::note
+:::warning
This is an experimental feature that is currently in development and is not ready for general use. It will change in unpredictable backwards-incompatible ways in future releases. Set `allow_experimental_nlp_functions = 1` to enable it.
:::
+## detectCharset
+
+The `detectCharset` function detects the character set of the non-UTF8-encoded input string.
+
+*Syntax*
+
+``` sql
+detectCharset('text_to_be_analyzed')
+```
+
+*Arguments*
+
+- `text_to_be_analyzed` — A collection (or sentences) of strings to analyze. [String](../data-types/string.md#string).
+
+*Returned value*
+
+- A `String` containing the code of the detected character set
+
+*Examples*
+
+Query:
+
+```sql
+SELECT detectCharset('Ich bleibe für ein paar Tage.');
+```
+
+Result:
+
+```response
+┌─detectCharset('Ich bleibe für ein paar Tage.')─┐
+│ WINDOWS-1252 │
+└────────────────────────────────────────────────┘
+```
+
+## detectLanguage
+
+Detects the language of the UTF8-encoded input string. The function uses the [CLD2 library](https://github.com/CLD2Owners/cld2) for detection, and it returns the 2-letter ISO language code.
+
+The `detectLanguage` function works best when providing over 200 characters in the input string.
+
+*Syntax*
+
+``` sql
+detectLanguage('text_to_be_analyzed')
+```
+
+*Arguments*
+
+- `text_to_be_analyzed` — A collection (or sentences) of strings to analyze. [String](../data-types/string.md#string).
+
+*Returned value*
+
+- The 2-letter ISO code of the detected language
+
+Other possible results:
+
+- `un` = unknown, can not detect any language.
+- `other` = the detected language does not have 2 letter code.
+
+*Examples*
+
+Query:
+
+```sql
+SELECT detectLanguage('Je pense que je ne parviendrai jamais à parler français comme un natif. Where there’s a will, there’s a way.');
+```
+
+Result:
+
+```response
+fr
+```
+
+## detectLanguageMixed
+
+Similar to the `detectLanguage` function, but `detectLanguageMixed` returns a `Map` of 2-letter language codes that are mapped to the percentage of the certain language in the text.
+
+
+*Syntax*
+
+``` sql
+detectLanguageMixed('text_to_be_analyzed')
+```
+
+*Arguments*
+
+- `text_to_be_analyzed` — A collection (or sentences) of strings to analyze. [String](../data-types/string.md#string).
+
+*Returned value*
+
+- `Map(String, Float32)`: The keys are 2-letter ISO codes and the values are a percentage of text found for that language
+
+
+*Examples*
+
+Query:
+
+```sql
+SELECT detectLanguageMixed('二兎を追う者は一兎をも得ず二兎を追う者は一兎をも得ず A vaincre sans peril, on triomphe sans gloire.');
+```
+
+Result:
+
+```response
+┌─detectLanguageMixed()─┐
+│ {'ja':0.62,'fr':0.36 │
+└───────────────────────┘
+```
+
+## detectProgrammingLanguage
+
+Determines the programming language from the source code. Calculates all the unigrams and bigrams of commands in the source code.
+Then using a marked-up dictionary with weights of unigrams and bigrams of commands for various programming languages finds the biggest weight of the programming language and returns it.
+
+*Syntax*
+
+``` sql
+detectProgrammingLanguage('source_code')
+```
+
+*Arguments*
+
+- `source_code` — String representation of the source code to analyze. [String](../data-types/string.md#string).
+
+*Returned value*
+
+- Programming language. [String](../data-types/string.md).
+
+*Examples*
+
+Query:
+
+```sql
+SELECT detectProgrammingLanguage('#include ');
+```
+
+Result:
+
+```response
+┌─detectProgrammingLanguage('#include ')─┐
+│ C++ │
+└──────────────────────────────────────────────────┘
+```
+
+## detectLanguageUnknown
+
+Similar to the `detectLanguage` function, except the `detectLanguageUnknown` function works with non-UTF8-encoded strings. Prefer this version when your character set is UTF-16 or UTF-32.
+
+
+*Syntax*
+
+``` sql
+detectLanguageUnknown('text_to_be_analyzed')
+```
+
+*Arguments*
+
+- `text_to_be_analyzed` — A collection (or sentences) of strings to analyze. [String](../data-types/string.md#string).
+
+*Returned value*
+
+- The 2-letter ISO code of the detected language
+
+Other possible results:
+
+- `un` = unknown, can not detect any language.
+- `other` = the detected language does not have 2 letter code.
+
+*Examples*
+
+Query:
+
+```sql
+SELECT detectLanguageUnknown('Ich bleibe für ein paar Tage.');
+```
+
+Result:
+
+```response
+┌─detectLanguageUnknown('Ich bleibe für ein paar Tage.')─┐
+│ de │
+└────────────────────────────────────────────────────────┘
+```
+
+## detectTonality
+
+Determines the sentiment of text data. Uses a marked-up sentiment dictionary, in which each word has a tonality ranging from `-12` to `6`.
+For each text, it calculates the average sentiment value of its words and returns it in the range `[-1,1]`.
+
+:::note
+This function is limited in its current form. Currently it makes use of the embedded emotional dictionary at `/contrib/nlp-data/tonality_ru.zst` and only works for the Russian language.
+:::
+
+*Syntax*
+
+``` sql
+detectTonality(text)
+```
+
+*Arguments*
+
+- `text` — The text to be analyzed. [String](../data-types/string.md#string).
+
+*Returned value*
+
+- The average sentiment value of the words in `text`. [Float32](../data-types/float.md).
+
+*Examples*
+
+Query:
+
+```sql
+SELECT detectTonality('Шарик - хороший пёс'), -- Sharik is a good dog
+ detectTonality('Шарик - пёс'), -- Sharik is a dog
+ detectTonality('Шарик - плохой пёс'); -- Sharkik is a bad dog
+```
+
+Result:
+
+```response
+┌─detectTonality('Шарик - хороший пёс')─┬─detectTonality('Шарик - пёс')─┬─detectTonality('Шарик - плохой пёс')─┐
+│ 0.44445 │ 0 │ -0.3 │
+└───────────────────────────────────────┴───────────────────────────────┴──────────────────────────────────────┘
+```
+## lemmatize
+
+Performs lemmatization on a given word. Needs dictionaries to operate, which can be obtained [here](https://github.com/vpodpecan/lemmagen3/tree/master/src/lemmagen3/models).
+
+*Syntax*
+
+``` sql
+lemmatize('language', word)
+```
+
+*Arguments*
+
+- `language` — Language which rules will be applied. [String](../data-types/string.md#string).
+- `word` — Word that needs to be lemmatized. Must be lowercase. [String](../data-types/string.md#string).
+
+*Examples*
+
+Query:
+
+``` sql
+SELECT lemmatize('en', 'wolves');
+```
+
+Result:
+
+``` text
+┌─lemmatize("wolves")─┐
+│ "wolf" │
+└─────────────────────┘
+```
+
+*Configuration*
+
+This configuration specifies that the dictionary `en.bin` should be used for lemmatization of English (`en`) words. The `.bin` files can be downloaded from
+[here](https://github.com/vpodpecan/lemmagen3/tree/master/src/lemmagen3/models).
+
+``` xml
+
+
+
+ en
+ en.bin
+
+
+
+```
+
## stem
Performs stemming on a given word.
-### Syntax
+*Syntax*
``` sql
stem('language', word)
```
-### Arguments
+*Arguments*
- `language` — Language which rules will be applied. Use the two letter [ISO 639-1 code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes).
- `word` — word that needs to be stemmed. Must be in lowercase. [String](../data-types/string.md#string).
-### Examples
+*Examples*
Query:
@@ -40,7 +311,7 @@ Result:
│ ['I','think','it','is','a','bless','in','disguis'] │
└────────────────────────────────────────────────────┘
```
-### Supported languages for stem()
+*Supported languages for stem()*
:::note
The stem() function uses the [Snowball stemming](https://snowballstem.org/) library, see the Snowball website for updated languages etc.
@@ -76,53 +347,6 @@ The stem() function uses the [Snowball stemming](https://snowballstem.org/) libr
- Turkish
- Yiddish
-## lemmatize
-
-Performs lemmatization on a given word. Needs dictionaries to operate, which can be obtained [here](https://github.com/vpodpecan/lemmagen3/tree/master/src/lemmagen3/models).
-
-### Syntax
-
-``` sql
-lemmatize('language', word)
-```
-
-### Arguments
-
-- `language` — Language which rules will be applied. [String](../data-types/string.md#string).
-- `word` — Word that needs to be lemmatized. Must be lowercase. [String](../data-types/string.md#string).
-
-### Examples
-
-Query:
-
-``` sql
-SELECT lemmatize('en', 'wolves');
-```
-
-Result:
-
-``` text
-┌─lemmatize("wolves")─┐
-│ "wolf" │
-└─────────────────────┘
-```
-
-### Configuration
-
-This configuration specifies that the dictionary `en.bin` should be used for lemmatization of English (`en`) words. The `.bin` files can be downloaded from
-[here](https://github.com/vpodpecan/lemmagen3/tree/master/src/lemmagen3/models).
-
-``` xml
-
-
-
- en
- en.bin
-
-
-
-```
-
## synonyms
Finds synonyms to a given word. There are two types of synonym extensions: `plain` and `wordnet`.
@@ -131,18 +355,18 @@ With the `plain` extension type we need to provide a path to a simple text file,
With the `wordnet` extension type we need to provide a path to a directory with WordNet thesaurus in it. Thesaurus must contain a WordNet sense index.
-### Syntax
+*Syntax*
``` sql
synonyms('extension_name', word)
```
-### Arguments
+*Arguments*
- `extension_name` — Name of the extension in which search will be performed. [String](../data-types/string.md#string).
- `word` — Word that will be searched in extension. [String](../data-types/string.md#string).
-### Examples
+*Examples*
Query:
@@ -158,7 +382,7 @@ Result:
└──────────────────────────────────────────┘
```
-### Configuration
+*Configuration*
``` xml
@@ -172,154 +396,4 @@ Result:
en/
-```
-
-## detectLanguage
-
-Detects the language of the UTF8-encoded input string. The function uses the [CLD2 library](https://github.com/CLD2Owners/cld2) for detection, and it returns the 2-letter ISO language code.
-
-The `detectLanguage` function works best when providing over 200 characters in the input string.
-
-### Syntax
-
-``` sql
-detectLanguage('text_to_be_analyzed')
-```
-
-### Arguments
-
-- `text_to_be_analyzed` — A collection (or sentences) of strings to analyze. [String](../data-types/string.md#string).
-
-### Returned value
-
-- The 2-letter ISO code of the detected language
-
-Other possible results:
-
-- `un` = unknown, can not detect any language.
-- `other` = the detected language does not have 2 letter code.
-
-### Examples
-
-Query:
-
-```sql
-SELECT detectLanguage('Je pense que je ne parviendrai jamais à parler français comme un natif. Where there’s a will, there’s a way.');
-```
-
-Result:
-
-```response
-fr
-```
-
-## detectLanguageMixed
-
-Similar to the `detectLanguage` function, but `detectLanguageMixed` returns a `Map` of 2-letter language codes that are mapped to the percentage of the certain language in the text.
-
-
-### Syntax
-
-``` sql
-detectLanguageMixed('text_to_be_analyzed')
-```
-
-### Arguments
-
-- `text_to_be_analyzed` — A collection (or sentences) of strings to analyze. [String](../data-types/string.md#string).
-
-### Returned value
-
-- `Map(String, Float32)`: The keys are 2-letter ISO codes and the values are a percentage of text found for that language
-
-
-### Examples
-
-Query:
-
-```sql
-SELECT detectLanguageMixed('二兎を追う者は一兎をも得ず二兎を追う者は一兎をも得ず A vaincre sans peril, on triomphe sans gloire.');
-```
-
-Result:
-
-```response
-┌─detectLanguageMixed()─┐
-│ {'ja':0.62,'fr':0.36 │
-└───────────────────────┘
-```
-
-## detectLanguageUnknown
-
-Similar to the `detectLanguage` function, except the `detectLanguageUnknown` function works with non-UTF8-encoded strings. Prefer this version when your character set is UTF-16 or UTF-32.
-
-
-### Syntax
-
-``` sql
-detectLanguageUnknown('text_to_be_analyzed')
-```
-
-### Arguments
-
-- `text_to_be_analyzed` — A collection (or sentences) of strings to analyze. [String](../data-types/string.md#string).
-
-### Returned value
-
-- The 2-letter ISO code of the detected language
-
-Other possible results:
-
-- `un` = unknown, can not detect any language.
-- `other` = the detected language does not have 2 letter code.
-
-### Examples
-
-Query:
-
-```sql
-SELECT detectLanguageUnknown('Ich bleibe für ein paar Tage.');
-```
-
-Result:
-
-```response
-┌─detectLanguageUnknown('Ich bleibe für ein paar Tage.')─┐
-│ de │
-└────────────────────────────────────────────────────────┘
-```
-
-## detectCharset
-
-The `detectCharset` function detects the character set of the non-UTF8-encoded input string.
-
-
-### Syntax
-
-``` sql
-detectCharset('text_to_be_analyzed')
-```
-
-### Arguments
-
-- `text_to_be_analyzed` — A collection (or sentences) of strings to analyze. [String](../data-types/string.md#string).
-
-### Returned value
-
-- A `String` containing the code of the detected character set
-
-### Examples
-
-Query:
-
-```sql
-SELECT detectCharset('Ich bleibe für ein paar Tage.');
-```
-
-Result:
-
-```response
-┌─detectCharset('Ich bleibe für ein paar Tage.')─┐
-│ WINDOWS-1252 │
-└────────────────────────────────────────────────┘
-```
+```
\ No newline at end of file
diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md
index 31df9e5627d..4e252785715 100644
--- a/docs/en/sql-reference/functions/other-functions.md
+++ b/docs/en/sql-reference/functions/other-functions.md
@@ -212,7 +212,7 @@ toTypeName(x)
## blockSize {#blockSize}
-In ClickHouse, queries are processed in blocks (chunks).
+In ClickHouse, queries are processed in [blocks](../../development/architecture.md/#block-block) (chunks).
This function returns the size (row count) of the block the function is called on.
**Syntax**
@@ -221,6 +221,33 @@ This function returns the size (row count) of the block the function is called o
blockSize()
```
+**Example**
+
+Query:
+
+```sql
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (n UInt8) ENGINE = Memory;
+
+INSERT INTO test
+SELECT * FROM system.numbers LIMIT 5;
+
+SELECT blockSize()
+FROM test;
+```
+
+Result:
+
+```response
+ ┌─blockSize()─┐
+1. │ 5 │
+2. │ 5 │
+3. │ 5 │
+4. │ 5 │
+5. │ 5 │
+ └─────────────┘
+```
+
## byteSize
Returns an estimation of uncompressed byte size of its arguments in memory.
@@ -735,7 +762,7 @@ LIMIT 10
Given a size (number of bytes), this function returns a readable, rounded size with suffix (KB, MB, etc.) as string.
-The opposite operations of this function are [parseReadableSize](#parseReadableSize), [parseReadableSizeOrZero](#parseReadableSizeOrZero), and [parseReadableSizeOrNull](#parseReadableSizeOrNull).
+The opposite operations of this function are [parseReadableSize](#parsereadablesize), [parseReadableSizeOrZero](#parsereadablesizeorzero), and [parseReadableSizeOrNull](#parsereadablesizeornull).
**Syntax**
@@ -768,7 +795,7 @@ Result:
Given a size (number of bytes), this function returns a readable, rounded size with suffix (KiB, MiB, etc.) as string.
-The opposite operations of this function are [parseReadableSize](#parseReadableSize), [parseReadableSizeOrZero](#parseReadableSizeOrZero), and [parseReadableSizeOrNull](#parseReadableSizeOrNull).
+The opposite operations of this function are [parseReadableSize](#parsereadablesize), [parseReadableSizeOrZero](#parsereadablesizeorzero), and [parseReadableSizeOrNull](#parsereadablesizeornull).
**Syntax**
@@ -899,7 +926,7 @@ SELECT
Given a string containing a byte size and `B`, `KiB`, `KB`, `MiB`, `MB`, etc. as a unit (i.e. [ISO/IEC 80000-13](https://en.wikipedia.org/wiki/ISO/IEC_80000) or decimal byte unit), this function returns the corresponding number of bytes.
If the function is unable to parse the input value, it throws an exception.
-The inverse operations of this function are [formatReadableSize](#formatReadableSize) and [formatReadableDecimalSize](#formatReadableDecimalSize).
+The inverse operations of this function are [formatReadableSize](#formatreadablesize) and [formatReadableDecimalSize](#formatreadabledecimalsize).
**Syntax**
@@ -937,7 +964,7 @@ SELECT
Given a string containing a byte size and `B`, `KiB`, `KB`, `MiB`, `MB`, etc. as a unit (i.e. [ISO/IEC 80000-13](https://en.wikipedia.org/wiki/ISO/IEC_80000) or decimal byte unit), this function returns the corresponding number of bytes.
If the function is unable to parse the input value, it returns `NULL`.
-The inverse operations of this function are [formatReadableSize](#formatReadableSize) and [formatReadableDecimalSize](#formatReadableDecimalSize).
+The inverse operations of this function are [formatReadableSize](#formatreadablesize) and [formatReadableDecimalSize](#formatreadabledecimalsize).
**Syntax**
@@ -975,7 +1002,7 @@ SELECT
Given a string containing a byte size and `B`, `KiB`, `KB`, `MiB`, `MB`, etc. as a unit (i.e. [ISO/IEC 80000-13](https://en.wikipedia.org/wiki/ISO/IEC_80000) or decimal byte unit), this function returns the corresponding number of bytes. If the function is unable to parse the input value, it returns `0`.
-The inverse operations of this function are [formatReadableSize](#formatReadableSize) and [formatReadableDecimalSize](#formatReadableDecimalSize).
+The inverse operations of this function are [formatReadableSize](#formatreadablesize) and [formatReadableDecimalSize](#formatreadabledecimalsize).
**Syntax**
@@ -2684,7 +2711,7 @@ countDigits(x)
- Number of digits. [UInt8](../data-types/int-uint.md#uint-ranges).
:::note
-For `Decimal` values takes into account their scales: calculates result over underlying integer type which is `(value * scale)`. For example: `countDigits(42) = 2`, `countDigits(42.000) = 5`, `countDigits(0.04200) = 4`. I.e. you may check decimal overflow for `Decimal64` with `countDecimal(x) > 18`. It's a slow variant of [isDecimalOverflow](#is-decimal-overflow).
+For `Decimal` values takes into account their scales: calculates result over underlying integer type which is `(value * scale)`. For example: `countDigits(42) = 2`, `countDigits(42.000) = 5`, `countDigits(0.04200) = 4`. I.e. you may check decimal overflow for `Decimal64` with `countDecimal(x) > 18`. It's a slow variant of [isDecimalOverflow](#isdecimaloverflow).
:::
**Example**
@@ -2776,7 +2803,7 @@ currentProfiles()
## enabledProfiles
-Returns settings profiles, assigned to the current user both explicitly and implicitly. Explicitly assigned profiles are the same as returned by the [currentProfiles](#current-profiles) function. Implicitly assigned profiles include parent profiles of other assigned profiles, profiles assigned via granted roles, profiles assigned via their own settings, and the main default profile (see the `default_profile` section in the main server configuration file).
+Returns settings profiles, assigned to the current user both explicitly and implicitly. Explicitly assigned profiles are the same as returned by the [currentProfiles](#currentprofiles) function. Implicitly assigned profiles include parent profiles of other assigned profiles, profiles assigned via granted roles, profiles assigned via their own settings, and the main default profile (see the `default_profile` section in the main server configuration file).
**Syntax**
@@ -2889,11 +2916,11 @@ Result:
└───────────────────────────┘
```
-## queryID {#queryID}
+## queryID
Returns the ID of the current query. Other parameters of a query can be extracted from the [system.query_log](../../operations/system-tables/query_log.md) table via `query_id`.
-In contrast to [initialQueryID](#initial-query-id) function, `queryID` can return different results on different shards (see the example).
+In contrast to [initialQueryID](#initialqueryid) function, `queryID` can return different results on different shards (see the example).
**Syntax**
@@ -2927,7 +2954,7 @@ Result:
Returns the ID of the initial current query. Other parameters of a query can be extracted from the [system.query_log](../../operations/system-tables/query_log.md) table via `initial_query_id`.
-In contrast to [queryID](#query-id) function, `initialQueryID` returns the same results on different shards (see example).
+In contrast to [queryID](#queryid) function, `initialQueryID` returns the same results on different shards (see example).
**Syntax**
@@ -3014,7 +3041,7 @@ shardCount()
**See Also**
-- [shardNum()](#shard-num) function example also contains `shardCount()` function call.
+- [shardNum()](#shardnum) function example also contains `shardCount()` function call.
## getOSKernelVersion
@@ -3688,3 +3715,283 @@ Result:
```response
{'version':'1','serial_number':'2D9071D64530052D48308473922C7ADAFA85D6C5','signature_algo':'sha256WithRSAEncryption','issuer':'/CN=marsnet.local CA','not_before':'May 7 17:01:21 2024 GMT','not_after':'May 7 17:01:21 2025 GMT','subject':'/CN=chnode1','pkey_algo':'rsaEncryption'}
```
+
+## lowCardinalityIndices
+
+Returns the position of a value in the dictionary of a [LowCardinality](../data-types/lowcardinality.md) column. Positions start at 1. Since LowCardinality have per-part dictionaries, this function may return different positions for the same value in different parts.
+
+**Syntax**
+
+```sql
+lowCardinalityIndices(col)
+```
+
+**Arguments**
+
+- `col` — a low cardinality column. [LowCardinality](../data-types/lowcardinality.md).
+
+**Returned value**
+
+- The position of the value in the dictionary of the current part. [UInt64](../data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+```sql
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (s LowCardinality(String)) ENGINE = Memory;
+
+-- create two parts:
+
+INSERT INTO test VALUES ('ab'), ('cd'), ('ab'), ('ab'), ('df');
+INSERT INTO test VALUES ('ef'), ('cd'), ('ab'), ('cd'), ('ef');
+
+SELECT s, lowCardinalityIndices(s) FROM test;
+```
+
+Result:
+
+```response
+ ┌─s──┬─lowCardinalityIndices(s)─┐
+1. │ ab │ 1 │
+2. │ cd │ 2 │
+3. │ ab │ 1 │
+4. │ ab │ 1 │
+5. │ df │ 3 │
+ └────┴──────────────────────────┘
+ ┌─s──┬─lowCardinalityIndices(s)─┐
+ 6. │ ef │ 1 │
+ 7. │ cd │ 2 │
+ 8. │ ab │ 3 │
+ 9. │ cd │ 2 │
+10. │ ef │ 1 │
+ └────┴──────────────────────────┘
+```
+## lowCardinalityKeys
+
+Returns the dictionary values of a [LowCardinality](../data-types/lowcardinality.md) column. If the block is smaller or larger than the dictionary size, the result will be truncated or extended with default values. Since LowCardinality have per-part dictionaries, this function may return different dictionary values in different parts.
+
+**Syntax**
+
+```sql
+lowCardinalityIndices(col)
+```
+
+**Arguments**
+
+- `col` — a low cardinality column. [LowCardinality](../data-types/lowcardinality.md).
+
+**Returned value**
+
+- The dictionary keys. [UInt64](../data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+```sql
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (s LowCardinality(String)) ENGINE = Memory;
+
+-- create two parts:
+
+INSERT INTO test VALUES ('ab'), ('cd'), ('ab'), ('ab'), ('df');
+INSERT INTO test VALUES ('ef'), ('cd'), ('ab'), ('cd'), ('ef');
+
+SELECT s, lowCardinalityKeys(s) FROM test;
+```
+
+Result:
+
+```response
+ ┌─s──┬─lowCardinalityKeys(s)─┐
+1. │ ef │ │
+2. │ cd │ ef │
+3. │ ab │ cd │
+4. │ cd │ ab │
+5. │ ef │ │
+ └────┴───────────────────────┘
+ ┌─s──┬─lowCardinalityKeys(s)─┐
+ 6. │ ab │ │
+ 7. │ cd │ ab │
+ 8. │ ab │ cd │
+ 9. │ ab │ df │
+10. │ df │ │
+ └────┴───────────────────────┘
+```
+
+## displayName
+
+Returns the value of `display_name` from [config](../../operations/configuration-files.md/#configuration-files) or server Fully Qualified Domain Name (FQDN) if not set.
+
+**Syntax**
+
+```sql
+displayName()
+```
+
+**Returned value**
+
+- Value of `display_name` from config or server FQDN if not set. [String](../data-types/string.md).
+
+**Example**
+
+The `display_name` can be set in `config.xml`. Taking for example a server with `display_name` configured to 'production':
+
+```xml
+
+production
+```
+
+Query:
+
+```sql
+SELECT displayName();
+```
+
+Result:
+
+```response
+┌─displayName()─┐
+│ production │
+└───────────────┘
+```
+
+## transactionID
+
+Returns the ID of a [transaction](https://clickhouse.com/docs/en/guides/developer/transactional#transactions-commit-and-rollback).
+
+:::note
+This function is part of an experimental feature set. Enable experimental transaction support by adding this setting to your configuration:
+
+```
+
+ 1
+
+```
+
+For more information see the page [Transactional (ACID) support](https://clickhouse.com/docs/en/guides/developer/transactional#transactions-commit-and-rollback).
+:::
+
+**Syntax**
+
+```sql
+transactionID()
+```
+
+**Returned value**
+
+- Returns a tuple consisting of `start_csn`, `local_tid` and `host_id`. [Tuple](../data-types/tuple.md).
+
+- `start_csn`: Global sequential number, the newest commit timestamp that was seen when this transaction began. [UInt64](../data-types/int-uint.md).
+- `local_tid`: Local sequential number that is unique for each transaction started by this host within a specific start_csn. [UInt64](../data-types/int-uint.md).
+- `host_id`: UUID of the host that has started this transaction. [UUID](../data-types/uuid.md).
+
+**Example**
+
+Query:
+
+```sql
+BEGIN TRANSACTION;
+SELECT transactionID();
+ROLLBACK;
+```
+
+Result:
+
+```response
+┌─transactionID()────────────────────────────────┐
+│ (32,34,'0ee8b069-f2bb-4748-9eae-069c85b5252b') │
+└────────────────────────────────────────────────┘
+```
+
+## transactionLatestSnapshot
+
+Returns the newest snapshot (Commit Sequence Number) of a [transaction](https://clickhouse.com/docs/en/guides/developer/transactional#transactions-commit-and-rollback) that is available for reading.
+
+:::note
+This function is part of an experimental feature set. Enable experimental transaction support by adding this setting to your configuration:
+
+```
+
+ 1
+
+```
+
+For more information see the page [Transactional (ACID) support](https://clickhouse.com/docs/en/guides/developer/transactional#transactions-commit-and-rollback).
+:::
+
+**Syntax**
+
+```sql
+transactionLatestSnapshot()
+```
+
+**Returned value**
+
+- Returns the latest snapshot (CSN) of a transaction. [UInt64](../data-types/int-uint.md)
+
+**Example**
+
+Query:
+
+```sql
+BEGIN TRANSACTION;
+SELECT transactionLatestSnapshot();
+ROLLBACK;
+```
+
+Result:
+
+```response
+┌─transactionLatestSnapshot()─┐
+│ 32 │
+└─────────────────────────────┘
+```
+
+## transactionOldestSnapshot
+
+Returns the oldest snapshot (Commit Sequence Number) that is visible for some running [transaction](https://clickhouse.com/docs/en/guides/developer/transactional#transactions-commit-and-rollback).
+
+:::note
+This function is part of an experimental feature set. Enable experimental transaction support by adding this setting to your configuration:
+
+```
+
+ 1
+
+```
+
+For more information see the page [Transactional (ACID) support](https://clickhouse.com/docs/en/guides/developer/transactional#transactions-commit-and-rollback).
+:::
+
+**Syntax**
+
+```sql
+transactionOldestSnapshot()
+```
+
+**Returned value**
+
+- Returns the oldest snapshot (CSN) of a transaction. [UInt64](../data-types/int-uint.md)
+
+**Example**
+
+Query:
+
+```sql
+BEGIN TRANSACTION;
+SELECT transactionLatestSnapshot();
+ROLLBACK;
+```
+
+Result:
+
+```response
+┌─transactionOldestSnapshot()─┐
+│ 32 │
+└─────────────────────────────┘
+```
diff --git a/docs/en/sql-reference/functions/rounding-functions.md b/docs/en/sql-reference/functions/rounding-functions.md
index d18185c5013..e2f471d47eb 100644
--- a/docs/en/sql-reference/functions/rounding-functions.md
+++ b/docs/en/sql-reference/functions/rounding-functions.md
@@ -6,49 +6,90 @@ sidebar_label: Rounding
# Rounding Functions
-## floor(x\[, N\])
+## floor
-Returns the largest round number that is less than or equal to `x`. A round number is a multiple of 1/10N, or the nearest number of the appropriate data type if 1 / 10N isn’t exact.
-‘N’ is an integer constant, optional parameter. By default it is zero, which means to round to an integer.
-‘N’ may be negative.
+Returns the largest rounded number less than or equal `x`.
+A rounded number is a multiple of 1 / 10 * N, or the nearest number of the appropriate data type if 1 / 10 * N isn’t exact.
-Examples: `floor(123.45, 1) = 123.4, floor(123.45, -1) = 120.`
+Integer arguments may be rounded with negative `N` argument, with non-negative `N` the function returns `x`, i.e. does nothing.
-`x` is any numeric type. The result is a number of the same type.
-For integer arguments, it makes sense to round with a negative `N` value (for non-negative `N`, the function does not do anything).
-If rounding causes overflow (for example, floor(-128, -1)), an implementation-specific result is returned.
+If rounding causes an overflow (for example, `floor(-128, -1)`), the result is undefined.
-## ceil(x\[, N\]), ceiling(x\[, N\])
+**Syntax**
-Returns the smallest round number that is greater than or equal to `x`. In every other way, it is the same as the `floor` function (see above).
+``` sql
+floor(x[, N])
+```
-## trunc(x\[, N\]), truncate(x\[, N\])
+**Parameters**
-Returns the round number with largest absolute value that has an absolute value less than or equal to `x`‘s. In every other way, it is the same as the ’floor’ function (see above).
+- `x` - The value to round. [Float*](../data-types/float.md), [Decimal*](../data-types/decimal.md), or [(U)Int*](../data-types/int-uint.md).
+- `N` . [(U)Int*](../data-types/int-uint.md). The default is zero, which means rounding to an integer. Can be negative.
+
+**Returned value**
+
+A rounded number of the same type as `x`.
+
+**Examples**
+
+Query:
+
+```sql
+SELECT floor(123.45, 1) AS rounded
+```
+
+Result:
+
+```
+┌─rounded─┐
+│ 123.4 │
+└─────────┘
+```
+
+Query:
+
+```sql
+SELECT floor(123.45, -1)
+```
+
+Result:
+
+```
+┌─rounded─┐
+│ 120 │
+└─────────┘
+```
+
+## ceiling
+
+Like `floor` but returns the smallest rounded number greater than or equal `x`.
+
+**Syntax**
+
+``` sql
+ceiling(x[, N])
+```
+
+Alias: `ceil`
+
+## truncate
+
+Like `floor` but returns the rounded number with largest absolute value that has an absolute value less than or equal to `x`‘s.
**Syntax**
```sql
-trunc(input, precision)
+truncate(x[, N])
```
-Alias: `truncate`.
-
-**Parameters**
-
-- `input`: A numeric type ([Float](../data-types/float.md), [Decimal](../data-types/decimal.md) or [Integer](../data-types/int-uint.md)).
-- `precision`: An [Integer](../data-types/int-uint.md) type.
-
-**Returned value**
-
-- A data type of `input`.
+Alias: `trunc`.
**Example**
Query:
```sql
-SELECT trunc(123.499, 1) as res;
+SELECT truncate(123.499, 1) as res;
```
```response
@@ -57,37 +98,40 @@ SELECT trunc(123.499, 1) as res;
└───────┘
```
-## round(x\[, N\])
+## round
Rounds a value to a specified number of decimal places.
-The function returns the nearest number of the specified order. In case when given number has equal distance to surrounding numbers, the function uses banker’s rounding for float number types and rounds away from zero for the other number types (Decimal).
+The function returns the nearest number of the specified order.
+If the input value has equal distance to two neighboring numbers, the function uses banker’s rounding for [Float*](../data-types/float.md) inputs and rounds away from zero for the other number types ([Decimal*](../data-types/decimal.md).
+
+**Syntax**
``` sql
-round(expression [, decimal_places])
+round(x[, N])
```
**Arguments**
-- `expression` — A number to be rounded. Can be any [expression](../../sql-reference/syntax.md#syntax-expressions) returning the numeric [data type](../data-types/index.md#data_types).
-- `decimal-places` — An integer value.
- - If `decimal-places > 0` then the function rounds the value to the right of the decimal point.
- - If `decimal-places < 0` then the function rounds the value to the left of the decimal point.
- - If `decimal-places = 0` then the function rounds the value to integer. In this case the argument can be omitted.
+- `x` — A number to round. [Float*](../data-types/float.md), [Decimal*](../data-types/decimal.md), or [(U)Int*](../data-types/int-uint.md).
+- `N` — The number of decimal places to round to. Integer. Defaults to `0`.
+ - If `N > 0`, the function rounds to the right of the decimal point.
+ - If `N < 0`, the function rounds to the left of the decimal point.
+ - If `N = 0`, the function rounds to the next integer.
**Returned value:**
-The rounded number of the same type as the input number.
+A rounded number of the same type as `x`.
**Examples**
-Example of usage with Float:
+Example with `Float` inputs:
-``` sql
+```sql
SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3;
```
-``` text
+```
┌───x─┬─round(divide(number, 2))─┐
│ 0 │ 0 │
│ 0.5 │ 0 │
@@ -95,13 +139,13 @@ SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3;
└─────┴──────────────────────────┘
```
-Example of usage with Decimal:
+Example with `Decimal` inputs:
-``` sql
+```sql
SELECT cast(number / 2 AS Decimal(10,4)) AS x, round(x) FROM system.numbers LIMIT 3;
```
-``` text
+```
┌───x─┬─round(CAST(divide(number, 2), 'Decimal(10, 4)'))─┐
│ 0 │ 0 │
│ 0.5 │ 1 │
@@ -109,14 +153,14 @@ SELECT cast(number / 2 AS Decimal(10,4)) AS x, round(x) FROM system.numbers LIM
└─────┴──────────────────────────────────────────────────┘
```
-If you want to keep the trailing zeros, you need to enable `output_format_decimal_trailing_zeros`
+To retain trailing zeros, enable setting `output_format_decimal_trailing_zeros`:
-``` sql
+```sql
SELECT cast(number / 2 AS Decimal(10,4)) AS x, round(x) FROM system.numbers LIMIT 3 settings output_format_decimal_trailing_zeros=1;
```
-``` text
+```
┌──────x─┬─round(CAST(divide(number, 2), 'Decimal(10, 4)'))─┐
│ 0.0000 │ 0.0000 │
│ 0.5000 │ 1.0000 │
@@ -151,9 +195,15 @@ round(3.65, 1) = 3.6
Rounds a number to a specified decimal position.
-- If the rounding number is halfway between two numbers, the function uses banker’s rounding. Banker's rounding is a method of rounding fractional numbers. When the rounding number is halfway between two numbers, it's rounded to the nearest even digit at the specified decimal position. For example: 3.5 rounds up to 4, 2.5 rounds down to 2. It's the default rounding method for floating point numbers defined in [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754#Roundings_to_nearest). The [round](#rounding_functions-round) function performs the same rounding for floating point numbers. The `roundBankers` function also rounds integers the same way, for example, `roundBankers(45, -1) = 40`.
+If the rounding number is halfway between two numbers, the function uses banker’s rounding.
+Banker's rounding is a method of rounding fractional numbers
+When the rounding number is halfway between two numbers, it's rounded to the nearest even digit at the specified decimal position.
+For example: 3.5 rounds up to 4, 2.5 rounds down to 2.
+It's the default rounding method for floating point numbers defined in [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754#Roundings_to_nearest).
+The [round](#round) function performs the same rounding for floating point numbers.
+The `roundBankers` function also rounds integers the same way, for example, `roundBankers(45, -1) = 40`.
-- In other cases, the function rounds numbers to the nearest integer.
+In other cases, the function rounds numbers to the nearest integer.
Using banker’s rounding, you can reduce the effect that rounding numbers has on the results of summing or subtracting these numbers.
@@ -166,16 +216,20 @@ For example, sum numbers 1.5, 2.5, 3.5, 4.5 with different rounding:
**Syntax**
``` sql
-roundBankers(expression [, decimal_places])
+roundBankers(x [, N])
```
**Arguments**
-- `expression` — A number to be rounded. Can be any [expression](../../sql-reference/syntax.md#syntax-expressions) returning the numeric [data type](../data-types/index.md#data_types).
-- `decimal-places` — Decimal places. An integer number.
- - `decimal-places > 0` — The function rounds the number to the given position right of the decimal point. Example: `roundBankers(3.55, 1) = 3.6`.
- - `decimal-places < 0` — The function rounds the number to the given position left of the decimal point. Example: `roundBankers(24.55, -1) = 20`.
- - `decimal-places = 0` — The function rounds the number to an integer. In this case the argument can be omitted. Example: `roundBankers(2.5) = 2`.
+ - `N > 0` — The function rounds the number to the given position right of the decimal point. Example: `roundBankers(3.55, 1) = 3.6`.
+ - `N < 0` — The function rounds the number to the given position left of the decimal point. Example: `roundBankers(24.55, -1) = 20`.
+ - `N = 0` — The function rounds the number to an integer. In this case the argument can be omitted. Example: `roundBankers(2.5) = 2`.
+
+- `x` — A number to round. [Float*](../data-types/float.md), [Decimal*](../data-types/decimal.md), or [(U)Int*](../data-types/int-uint.md).
+- `N` — The number of decimal places to round to. Integer. Defaults to `0`.
+ - If `N > 0`, the function rounds to the right of the decimal point.
+ - If `N < 0`, the function rounds to the left of the decimal point.
+ - If `N = 0`, the function rounds to the next integer.
**Returned value**
@@ -185,13 +239,13 @@ A value rounded by the banker’s rounding method.
Query:
-``` sql
+```sql
SELECT number / 2 AS x, roundBankers(x, 0) AS b fROM system.numbers limit 10
```
Result:
-``` text
+```
┌───x─┬─b─┐
│ 0 │ 0 │
│ 0.5 │ 0 │
@@ -208,7 +262,7 @@ Result:
Examples of Banker’s rounding:
-``` text
+```
roundBankers(0.4) = 0
roundBankers(-3.5) = -4
roundBankers(4.5) = 4
@@ -220,7 +274,7 @@ roundBankers(10.755, 2) = 10.76
**See Also**
-- [round](#rounding_functions-round)
+- [round](#round)
## roundToExp2
@@ -264,7 +318,7 @@ Result:
## roundDuration
-Accepts a number. If the number is less than one, it returns `0`. Otherwise, it rounds the number down to numbers from the set of commonly used durations: `1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000`.
+Accepts a number. If the number is less than one, it returns `0`. Otherwise, it rounds the number down to numbers from the set of commonly used durations: `1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000`.
**Syntax**
diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md
index 342ca2b9f03..894b9026165 100644
--- a/docs/en/sql-reference/functions/string-functions.md
+++ b/docs/en/sql-reference/functions/string-functions.md
@@ -579,7 +579,6 @@ If the length of the UTF-8 byte sequence is different for upper and lower case o
Converts a string to uppercase, assuming that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
-Does not detect the language, e.g. for Turkish the result might not be exactly correct (i/İ vs. i/I).
If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point.
@@ -736,7 +735,7 @@ concat(s1, s2, ...)
**Arguments**
-At least one value of arbitrary type.
+Values of arbitrary type.
Arguments which are not of types [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md) are converted to strings using their default serialization. As this decreases performance, it is not recommended to use non-String/FixedString arguments.
@@ -1136,16 +1135,136 @@ SELECT tryBase58Decode('3dc8KtHrwM') as res, tryBase58Decode('invalid') as res_i
## base64Encode
-Encodes a String or FixedString as base64.
+Encodes a String or FixedString as base64, according to [RFC 4648](https://datatracker.ietf.org/doc/html/rfc4648#section-4).
Alias: `TO_BASE64`.
+**Syntax**
+
+```sql
+base64Encode(plaintext)
+```
+
+**Arguments**
+
+- `plaintext` — [String](../data-types/string.md) column or constant.
+
+**Returned value**
+
+- A string containing the encoded value of the argument.
+
+**Example**
+
+``` sql
+SELECT base64Encode('clickhouse');
+```
+
+Result:
+
+```result
+┌─base64Encode('clickhouse')─┐
+│ Y2xpY2tob3VzZQ== │
+└────────────────────────────┘
+```
+
+## base64URLEncode
+
+Encodes an URL (String or FixedString) as base64 with URL-specific modifications, according to [RFC 4648](https://datatracker.ietf.org/doc/html/rfc4648#section-5).
+
+**Syntax**
+
+```sql
+base64URLEncode(url)
+```
+
+**Arguments**
+
+- `url` — [String](../data-types/string.md) column or constant.
+
+**Returned value**
+
+- A string containing the encoded value of the argument.
+
+**Example**
+
+``` sql
+SELECT base64URLEncode('https://clickhouse.com');
+```
+
+Result:
+
+```result
+┌─base64URLEncode('https://clickhouse.com')─┐
+│ aHR0cDovL2NsaWNraG91c2UuY29t │
+└───────────────────────────────────────────┘
+```
+
## base64Decode
-Decodes a base64-encoded String or FixedString. Throws an exception in case of error.
+Accepts a String and decodes it from base64, according to [RFC 4648](https://datatracker.ietf.org/doc/html/rfc4648#section-4). Throws an exception in case of an error.
Alias: `FROM_BASE64`.
+**Syntax**
+
+```sql
+base64Decode(encoded)
+```
+
+**Arguments**
+
+- `encoded` — [String](../data-types/string.md) column or constant. If the string is not a valid Base64-encoded value, an exception is thrown.
+
+**Returned value**
+
+- A string containing the decoded value of the argument.
+
+**Example**
+
+``` sql
+SELECT base64Decode('Y2xpY2tob3VzZQ==');
+```
+
+Result:
+
+```result
+┌─base64Decode('Y2xpY2tob3VzZQ==')─┐
+│ clickhouse │
+└──────────────────────────────────┘
+```
+
+## base64URLDecode
+
+Accepts a base64-encoded URL and decodes it from base64 with URL-specific modifications, according to [RFC 4648](https://datatracker.ietf.org/doc/html/rfc4648#section-5). Throws an exception in case of an error.
+
+**Syntax**
+
+```sql
+base64URLDecode(encodedUrl)
+```
+
+**Arguments**
+
+- `encodedURL` — [String](../data-types/string.md) column or constant. If the string is not a valid Base64-encoded value with URL-specific modifications, an exception is thrown.
+
+**Returned value**
+
+- A string containing the decoded value of the argument.
+
+**Example**
+
+``` sql
+SELECT base64URLDecode('aHR0cDovL2NsaWNraG91c2UuY29t');
+```
+
+Result:
+
+```result
+┌─base64URLDecode('aHR0cDovL2NsaWNraG91c2UuY29t')─┐
+│ https://clickhouse.com │
+└─────────────────────────────────────────────────┘
+```
+
## tryBase64Decode
Like `base64Decode` but returns an empty string in case of error.
@@ -1156,9 +1275,13 @@ Like `base64Decode` but returns an empty string in case of error.
tryBase64Decode(encoded)
```
-**Parameters**
+**Arguments**
-- `encoded`: [String](../data-types/string.md) column or constant. If the string is not a valid Base58-encoded value, returns an empty string in case of error.
+- `encoded`: [String](../data-types/string.md) column or constant. If the string is not a valid Base64-encoded value, returns an empty string.
+
+**Returned value**
+
+- A string containing the decoded value of the argument.
**Examples**
@@ -1169,9 +1292,41 @@ SELECT tryBase64Decode('RW5jb2RlZA==') as res, tryBase64Decode('invalid') as res
```
```response
-┌─res─────┬─res_invalid─┐
-│ Encoded │ │
-└─────────┴─────────────┘
+┌─res────────┬─res_invalid─┐
+│ clickhouse │ │
+└────────────┴─────────────┘
+```
+
+## tryBase64URLDecode
+
+Like `base64URLDecode` but returns an empty string in case of error.
+
+**Syntax**
+
+```sql
+tryBase64URLDecode(encodedUrl)
+```
+
+**Parameters**
+
+- `encodedURL`: [String](../data-types/string.md) column or constant. If the string is not a valid Base64-encoded value with URL-specific modifications, returns an empty string.
+
+**Returned value**
+
+- A string containing the decoded value of the argument.
+
+**Examples**
+
+Query:
+
+```sql
+SELECT tryBase64URLDecode('aHR0cDovL2NsaWNraG91c2UuY29t') as res, tryBase64Decode('aHR0cHM6Ly9jbGlja') as res_invalid;
+```
+
+```response
+┌─res────────────────────┬─res_invalid─┐
+│ https://clickhouse.com │ │
+└────────────────────────┴─────────────┘
```
## endsWith {#endswith}
@@ -1994,7 +2149,7 @@ Result:
## stringJaccardIndexUTF8
-Like [stringJaccardIndex](#stringJaccardIndex) but for UTF8-encoded strings.
+Like [stringJaccardIndex](#stringjaccardindex) but for UTF8-encoded strings.
## editDistance
@@ -2022,6 +2177,32 @@ Result:
Alias: levenshteinDistance
+## editDistanceUTF8
+
+Calculates the [edit distance](https://en.wikipedia.org/wiki/Edit_distance) between two UTF8 strings.
+
+**Syntax**
+
+```sql
+editDistanceUTF8(string1, string2)
+```
+
+**Examples**
+
+``` sql
+SELECT editDistanceUTF8('我是谁', '我是我');
+```
+
+Result:
+
+``` text
+┌─editDistanceUTF8('我是谁', '我是我')──┐
+│ 1 │
+└─────────────────────────────────────┘
+```
+
+Alias: levenshteinDistanceUTF8
+
## damerauLevenshteinDistance
Calculates the [Damerau-Levenshtein distance](https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance) between two byte strings.
diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md
index d261cff3580..b7ba1d4feb7 100644
--- a/docs/en/sql-reference/functions/string-search-functions.md
+++ b/docs/en/sql-reference/functions/string-search-functions.md
@@ -262,7 +262,7 @@ Result:
## multiSearchAllPositionsUTF8
-Like [multiSearchAllPositions](#multiSearchAllPositions) but assumes `haystack` and the `needle` substrings are UTF-8 encoded strings.
+Like [multiSearchAllPositions](#multisearchallpositions) but assumes `haystack` and the `needle` substrings are UTF-8 encoded strings.
**Syntax**
@@ -336,7 +336,7 @@ Result:
Like [`position`](#position) but returns the leftmost offset in a `haystack` string which matches any of multiple `needle` strings.
-Functions [`multiSearchFirstPositionCaseInsensitive`](#multiSearchFirstPositionCaseInsensitive), [`multiSearchFirstPositionUTF8`](#multiSearchFirstPositionUTF8) and [`multiSearchFirstPositionCaseInsensitiveUTF8`](#multiSearchFirstPositionCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function.
+Functions [`multiSearchFirstPositionCaseInsensitive`](#multisearchfirstpositioncaseinsensitive), [`multiSearchFirstPositionUTF8`](#multisearchfirstpositionutf8) and [`multiSearchFirstPositionCaseInsensitiveUTF8`](#multisearchfirstpositioncaseinsensitiveutf8) provide case-insensitive and/or UTF-8 variants of this function.
**Syntax**
@@ -370,7 +370,7 @@ Result:
## multiSearchFirstPositionCaseInsensitive
-Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but ignores case.
+Like [`multiSearchFirstPosition`](#multisearchfirstposition) but ignores case.
**Syntax**
@@ -404,7 +404,7 @@ Result:
## multiSearchFirstPositionUTF8
-Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but assumes `haystack` and `needle` to be UTF-8 strings.
+Like [`multiSearchFirstPosition`](#multisearchfirstposition) but assumes `haystack` and `needle` to be UTF-8 strings.
**Syntax**
@@ -440,7 +440,7 @@ Result:
## multiSearchFirstPositionCaseInsensitiveUTF8
-Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but assumes `haystack` and `needle` to be UTF-8 strings and ignores case.
+Like [`multiSearchFirstPosition`](#multisearchfirstposition) but assumes `haystack` and `needle` to be UTF-8 strings and ignores case.
**Syntax**
@@ -478,7 +478,7 @@ Result:
Returns the index `i` (starting from 1) of the leftmost found needlei in the string `haystack` and 0 otherwise.
-Functions [`multiSearchFirstIndexCaseInsensitive`](#multiSearchFirstIndexCaseInsensitive), [`multiSearchFirstIndexUTF8`](#multiSearchFirstIndexUTF8) and [`multiSearchFirstIndexCaseInsensitiveUTF8`](#multiSearchFirstIndexCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function.
+Functions [`multiSearchFirstIndexCaseInsensitive`](#multisearchfirstindexcaseinsensitive), [`multiSearchFirstIndexUTF8`](#multisearchfirstindexutf8) and [`multiSearchFirstIndexCaseInsensitiveUTF8`](#multisearchfirstindexcaseinsensitiveutf8) provide case-insensitive and/or UTF-8 variants of this function.
**Syntax**
@@ -615,7 +615,7 @@ Result:
Returns 1, if at least one string needlei matches the string `haystack` and 0 otherwise.
-Functions [`multiSearchAnyCaseInsensitive`](#multiSearchAnyCaseInsensitive), [`multiSearchAnyUTF8`](#multiSearchAnyUTF8) and []`multiSearchAnyCaseInsensitiveUTF8`](#multiSearchAnyCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function.
+Functions [`multiSearchAnyCaseInsensitive`](#multisearchanycaseinsensitive), [`multiSearchAnyUTF8`](#multisearchanyutf8) and [`multiSearchAnyCaseInsensitiveUTF8`](#multisearchanycaseinsensitiveutf8) provide case-insensitive and/or UTF-8 variants of this function.
**Syntax**
@@ -719,7 +719,7 @@ Result:
## multiSearchAnyCaseInsensitiveUTF8
-Like [multiSearchAnyUTF8](#multiSearchAnyUTF8) but ignores case.
+Like [multiSearchAnyUTF8](#multisearchanyutf8) but ignores case.
*Syntax**
@@ -880,7 +880,7 @@ extractAll(haystack, pattern)
Matches all groups of the `haystack` string using the `pattern` regular expression. Returns an array of arrays, where the first array includes all fragments matching the first group, the second array - matching the second group, etc.
-This function is slower than [extractAllGroupsVertical](#extractallgroups-vertical).
+This function is slower than [extractAllGroupsVertical](#extractallgroupsvertical).
**Syntax**
@@ -952,7 +952,7 @@ Result:
└────────────────────────────────────────────────────────────────────────────────────────┘
```
-## like {#like}
+## like
Returns whether string `haystack` matches the LIKE expression `pattern`.
@@ -1215,7 +1215,7 @@ Result:
## ngramSearchCaseInsensitive
-Provides a case-insensitive variant of [ngramSearch](#ngramSearch).
+Provides a case-insensitive variant of [ngramSearch](#ngramsearch).
**Syntax**
@@ -1630,7 +1630,7 @@ Result:
## hasSubsequenceCaseInsensitive
-Like [hasSubsequence](#hasSubsequence) but searches case-insensitively.
+Like [hasSubsequence](#hassubsequence) but searches case-insensitively.
**Syntax**
@@ -1665,7 +1665,7 @@ Result:
## hasSubsequenceUTF8
-Like [hasSubsequence](#hasSubsequence) but assumes `haystack` and `needle` are UTF-8 encoded strings.
+Like [hasSubsequence](#hassubsequence) but assumes `haystack` and `needle` are UTF-8 encoded strings.
**Syntax**
@@ -1700,7 +1700,7 @@ Result:
## hasSubsequenceCaseInsensitiveUTF8
-Like [hasSubsequenceUTF8](#hasSubsequenceUTF8) but searches case-insensitively.
+Like [hasSubsequenceUTF8](#hassubsequenceutf8) but searches case-insensitively.
**Syntax**
diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 5dd1d5ceebe..61e84ca72d1 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -10,7 +10,7 @@ sidebar_label: Type Conversion
ClickHouse generally uses the [same behavior as C++ programs](https://en.cppreference.com/w/cpp/language/implicit_conversion).
-`to` functions and [cast](#castx-t) behave differently in some cases, for example in case of [LowCardinality](../data-types/lowcardinality.md): [cast](#castx-t) removes [LowCardinality](../data-types/lowcardinality.md) trait `to` functions don't. The same with [Nullable](../data-types/nullable.md), this behaviour is not compatible with SQL standard, and it can be changed using [cast_keep_nullable](../../operations/settings/settings.md/#cast_keep_nullable) setting.
+`to` functions and [cast](#cast) behave differently in some cases, for example in case of [LowCardinality](../data-types/lowcardinality.md): [cast](#cast) removes [LowCardinality](../data-types/lowcardinality.md) trait `to` functions don't. The same with [Nullable](../data-types/nullable.md), this behaviour is not compatible with SQL standard, and it can be changed using [cast_keep_nullable](../../operations/settings/settings.md/#cast_keep_nullable) setting.
:::note
Be aware of potential data loss if values of a datatype are converted to a smaller datatype (for example from `Int64` to `Int32`) or between
@@ -70,7 +70,7 @@ Integer value in the `Int8`, `Int16`, `Int32`, `Int64`, `Int128` or `Int256` dat
Functions use [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning they truncate fractional digits of numbers.
-The behavior of functions for the [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments is undefined. Remember about [numeric conversions issues](#numeric-conversion-issues), when using the functions.
+The behavior of functions for the [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments is undefined. Remember about [numeric conversions issues](#common-issues-with-data-conversion), when using the functions.
**Example**
@@ -169,7 +169,7 @@ Converts an input value to the [UInt](../data-types/int-uint.md) data type. This
Functions use [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning they truncate fractional digits of numbers.
-The behavior of functions for negative arguments and for the [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments is undefined. If you pass a string with a negative number, for example `'-32'`, ClickHouse raises an exception. Remember about [numeric conversions issues](#numeric-conversion-issues), when using the functions.
+The behavior of functions for negative arguments and for the [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments is undefined. If you pass a string with a negative number, for example `'-32'`, ClickHouse raises an exception. Remember about [numeric conversions issues](#common-issues-with-data-conversion), when using the functions.
**Example**
@@ -996,7 +996,7 @@ Result:
## reinterpretAsUInt8
-Performs byte reinterpretation by treating the input value as a value of type UInt8. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type UInt8. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1034,7 +1034,7 @@ Result:
## reinterpretAsUInt16
-Performs byte reinterpretation by treating the input value as a value of type UInt16. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type UInt16. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1072,7 +1072,7 @@ Result:
## reinterpretAsUInt32
-Performs byte reinterpretation by treating the input value as a value of type UInt32. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type UInt32. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1110,7 +1110,7 @@ Result:
## reinterpretAsUInt64
-Performs byte reinterpretation by treating the input value as a value of type UInt64. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type UInt64. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1148,7 +1148,7 @@ Result:
## reinterpretAsUInt128
-Performs byte reinterpretation by treating the input value as a value of type UInt128. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type UInt128. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1186,7 +1186,7 @@ Result:
## reinterpretAsUInt256
-Performs byte reinterpretation by treating the input value as a value of type UInt256. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type UInt256. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1224,7 +1224,7 @@ Result:
## reinterpretAsInt8
-Performs byte reinterpretation by treating the input value as a value of type Int8. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type Int8. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1262,7 +1262,7 @@ Result:
## reinterpretAsInt16
-Performs byte reinterpretation by treating the input value as a value of type Int16. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type Int16. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1300,7 +1300,7 @@ Result:
## reinterpretAsInt32
-Performs byte reinterpretation by treating the input value as a value of type Int32. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type Int32. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1338,7 +1338,7 @@ Result:
## reinterpretAsInt64
-Performs byte reinterpretation by treating the input value as a value of type Int64. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type Int64. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1376,7 +1376,7 @@ Result:
## reinterpretAsInt128
-Performs byte reinterpretation by treating the input value as a value of type Int128. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type Int128. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1414,7 +1414,7 @@ Result:
## reinterpretAsInt256
-Performs byte reinterpretation by treating the input value as a value of type Int256. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type Int256. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1452,7 +1452,7 @@ Result:
## reinterpretAsFloat32
-Performs byte reinterpretation by treating the input value as a value of type Float32. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type Float32. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1486,7 +1486,7 @@ Result:
## reinterpretAsFloat64
-Performs byte reinterpretation by treating the input value as a value of type Float64. Unlike [`CAST`](#castx-t), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
+Performs byte reinterpretation by treating the input value as a value of type Float64. Unlike [`CAST`](#cast), the function does not attempt to preserve the original value - if the target type is not able to represent the input type, the output is meaningless.
**Syntax**
@@ -1730,7 +1730,7 @@ Result:
└─────────────────────┘
```
-## reinterpret(x, T)
+## reinterpret
Uses the same source in-memory bytes sequence for `x` value and reinterprets it to destination type.
@@ -1766,9 +1766,9 @@ Result:
└─────────────┴──────────────┴───────────────┘
```
-## CAST(x, T)
+## CAST
-Converts an input value to the specified data type. Unlike the [reinterpret](#type_conversion_function-reinterpret) function, `CAST` tries to present the same value using the new data type. If the conversion can not be done then an exception is raised.
+Converts an input value to the specified data type. Unlike the [reinterpret](#reinterpret) function, `CAST` tries to present the same value using the new data type. If the conversion can not be done then an exception is raised.
Several syntax variants are supported.
**Syntax**
@@ -1875,7 +1875,7 @@ Result:
Converts `x` to the `T` data type.
-The difference from [cast(x, T)](#type_conversion_function-cast) is that `accurateCast` does not allow overflow of numeric types during cast if type value `x` does not fit the bounds of type `T`. For example, `accurateCast(-1, 'UInt8')` throws an exception.
+The difference from [cast](#cast) is that `accurateCast` does not allow overflow of numeric types during cast if type value `x` does not fit the bounds of type `T`. For example, `accurateCast(-1, 'UInt8')` throws an exception.
**Example**
@@ -2061,7 +2061,7 @@ Result:
└───────────────────────────┴──────────────────────────────┘
```
-## parseDateTime {#type_conversion_functions-parseDateTime}
+## parseDateTime
Converts a [String](../data-types/string.md) to [DateTime](../data-types/datetime.md) according to a [MySQL format string](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_date-format).
@@ -2102,15 +2102,15 @@ Alias: `TO_TIMESTAMP`.
## parseDateTimeOrZero
-Same as for [parseDateTime](#type_conversion_functions-parseDateTime) except that it returns zero date when it encounters a date format that cannot be processed.
+Same as for [parseDateTime](#parsedatetime) except that it returns zero date when it encounters a date format that cannot be processed.
## parseDateTimeOrNull
-Same as for [parseDateTime](#type_conversion_functions-parseDateTime) except that it returns `NULL` when it encounters a date format that cannot be processed.
+Same as for [parseDateTime](#parsedatetime) except that it returns `NULL` when it encounters a date format that cannot be processed.
Alias: `str_to_date`.
-## parseDateTimeInJodaSyntax {#type_conversion_functions-parseDateTimeInJodaSyntax}
+## parseDateTimeInJodaSyntax
Similar to [parseDateTime](#parsedatetime), except that the format string is in [Joda](https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html) instead of MySQL syntax.
@@ -2151,11 +2151,11 @@ SELECT parseDateTimeInJodaSyntax('2023-02-24 14:53:31', 'yyyy-MM-dd HH:mm:ss', '
## parseDateTimeInJodaSyntaxOrZero
-Same as for [parseDateTimeInJodaSyntax](#type_conversion_functions-parseDateTimeInJodaSyntax) except that it returns zero date when it encounters a date format that cannot be processed.
+Same as for [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax) except that it returns zero date when it encounters a date format that cannot be processed.
## parseDateTimeInJodaSyntaxOrNull
-Same as for [parseDateTimeInJodaSyntax](#type_conversion_functions-parseDateTimeInJodaSyntax) except that it returns `NULL` when it encounters a date format that cannot be processed.
+Same as for [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax) except that it returns `NULL` when it encounters a date format that cannot be processed.
## parseDateTimeBestEffort
## parseDateTime32BestEffort
@@ -2313,11 +2313,11 @@ Same as for [parseDateTimeBestEffort](#parsedatetimebesteffort) except that it r
## parseDateTimeBestEffortUSOrNull
-Same as [parseDateTimeBestEffortUS](#parsedatetimebesteffortUS) function except that it returns `NULL` when it encounters a date format that cannot be processed.
+Same as [parseDateTimeBestEffortUS](#parsedatetimebesteffortus) function except that it returns `NULL` when it encounters a date format that cannot be processed.
## parseDateTimeBestEffortUSOrZero
-Same as [parseDateTimeBestEffortUS](#parsedatetimebesteffortUS) function except that it returns zero date (`1970-01-01`) or zero date with time (`1970-01-01 00:00:00`) when it encounters a date format that cannot be processed.
+Same as [parseDateTimeBestEffortUS](#parsedatetimebesteffortus) function except that it returns zero date (`1970-01-01`) or zero date with time (`1970-01-01 00:00:00`) when it encounters a date format that cannot be processed.
## parseDateTime64BestEffort
@@ -2389,7 +2389,7 @@ Same as for [parseDateTime64BestEffort](#parsedatetime64besteffort), except that
Converts input parameter to the [LowCardinality](../data-types/lowcardinality.md) version of same data type.
-To convert data from the `LowCardinality` data type use the [CAST](#type_conversion_function-cast) function. For example, `CAST(x as String)`.
+To convert data from the `LowCardinality` data type use the [CAST](#cast) function. For example, `CAST(x as String)`.
**Syntax**
@@ -2423,11 +2423,7 @@ Result:
## toUnixTimestamp64Milli
-## toUnixTimestamp64Micro
-
-## toUnixTimestamp64Nano
-
-Converts a `DateTime64` to a `Int64` value with fixed sub-second precision. Input value is scaled up or down appropriately depending on it precision.
+Converts a `DateTime64` to a `Int64` value with fixed millisecond precision. The input value is scaled up or down appropriately depending on its precision.
:::note
The output value is a timestamp in UTC, not in the timezone of `DateTime64`.
@@ -2437,24 +2433,22 @@ The output value is a timestamp in UTC, not in the timezone of `DateTime64`.
```sql
toUnixTimestamp64Milli(value)
-toUnixTimestamp64Micro(value)
-toUnixTimestamp64Nano(value)
```
**Arguments**
-- `value` — DateTime64 value with any precision.
+- `value` — DateTime64 value with any precision. [DateTime64](../data-types/datetime64.md).
**Returned value**
-- `value` converted to the `Int64` data type.
+- `value` converted to the `Int64` data type. [Int64](../data-types/int-uint.md).
-**Examples**
+**Example**
Query:
```sql
-WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64
+WITH toDateTime64('2009-02-13 23:31:31.011', 3, 'UTC') AS dt64
SELECT toUnixTimestamp64Milli(dt64);
```
@@ -2462,14 +2456,77 @@ Result:
```response
┌─toUnixTimestamp64Milli(dt64)─┐
-│ 1568650812345 │
+│ 1234567891011 │
└──────────────────────────────┘
```
+## toUnixTimestamp64Micro
+
+Converts a `DateTime64` to a `Int64` value with fixed microsecond precision. The input value is scaled up or down appropriately depending on its precision.
+
+:::note
+The output value is a timestamp in UTC, not in the timezone of `DateTime64`.
+:::
+
+**Syntax**
+
+```sql
+toUnixTimestamp64Micro(value)
+```
+
+**Arguments**
+
+- `value` — DateTime64 value with any precision. [DateTime64](../data-types/datetime64.md).
+
+**Returned value**
+
+- `value` converted to the `Int64` data type. [Int64](../data-types/int-uint.md).
+
+**Example**
+
Query:
-``` sql
-WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64
+```sql
+WITH toDateTime64('1970-01-15 06:56:07.891011', 6, 'UTC') AS dt64
+SELECT toUnixTimestamp64Micro(dt64);
+```
+
+Result:
+
+```response
+┌─toUnixTimestamp64Micro(dt64)─┐
+│ 1234567891011 │
+└──────────────────────────────┘
+```
+
+## toUnixTimestamp64Nano
+
+Converts a `DateTime64` to a `Int64` value with fixed nanosecond precision. The input value is scaled up or down appropriately depending on its precision.
+
+:::note
+The output value is a timestamp in UTC, not in the timezone of `DateTime64`.
+:::
+
+**Syntax**
+
+```sql
+toUnixTimestamp64Nano(value)
+```
+
+**Arguments**
+
+- `value` — DateTime64 value with any precision. [DateTime64](../data-types/datetime64.md).
+
+**Returned value**
+
+- `value` converted to the `Int64` data type. [Int64](../data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+```sql
+WITH toDateTime64('1970-01-01 00:20:34.567891011', 9, 'UTC') AS dt64
SELECT toUnixTimestamp64Nano(dt64);
```
@@ -2477,34 +2534,32 @@ Result:
```response
┌─toUnixTimestamp64Nano(dt64)─┐
-│ 1568650812345678000 │
+│ 1234567891011 │
└─────────────────────────────┘
```
## fromUnixTimestamp64Milli
-## fromUnixTimestamp64Micro
+Converts an `Int64` to a `DateTime64` value with fixed millisecond precision and optional timezone. The input value is scaled up or down appropriately depending on its precision.
-## fromUnixTimestamp64Nano
-
-Converts an `Int64` to a `DateTime64` value with fixed sub-second precision and optional timezone. Input value is scaled up or down appropriately depending on it’s precision. Please note that input value is treated as UTC timestamp, not timestamp at given (or implicit) timezone.
+:::note
+Please note that input value is treated as a UTC timestamp, not timestamp at the given (or implicit) timezone.
+:::
**Syntax**
``` sql
fromUnixTimestamp64Milli(value[, timezone])
-fromUnixTimestamp64Micro(value[, timezone])
-fromUnixTimestamp64Nano(value[, timezone])
```
**Arguments**
-- `value` — `Int64` value with any precision.
-- `timezone` — `String` (optional) timezone name of the result.
+- `value` — value with any precision. [Int64](../data-types/int-uint.md).
+- `timezone` — (optional) timezone name of the result. [String](../data-types/string.md).
**Returned value**
-- `value` converted to the `DateTime64` data type.
+- `value` converted to DateTime64 with precision `3`. [DateTime64](../data-types/datetime64.md).
**Example**
@@ -2512,15 +2567,101 @@ Query:
``` sql
WITH CAST(1234567891011, 'Int64') AS i64
-SELECT fromUnixTimestamp64Milli(i64, 'UTC');
+SELECT
+ fromUnixTimestamp64Milli(i64, 'UTC') AS x,
+ toTypeName(x);
```
Result:
```response
-┌─fromUnixTimestamp64Milli(i64, 'UTC')─┐
-│ 2009-02-13 23:31:31.011 │
-└──────────────────────────────────────┘
+┌───────────────────────x─┬─toTypeName(x)────────┐
+│ 2009-02-13 23:31:31.011 │ DateTime64(3, 'UTC') │
+└─────────────────────────┴──────────────────────┘
+```
+
+## fromUnixTimestamp64Micro
+
+Converts an `Int64` to a `DateTime64` value with fixed microsecond precision and optional timezone. The input value is scaled up or down appropriately depending on its precision.
+
+:::note
+Please note that input value is treated as a UTC timestamp, not timestamp at the given (or implicit) timezone.
+:::
+
+**Syntax**
+
+``` sql
+fromUnixTimestamp64Micro(value[, timezone])
+```
+
+**Arguments**
+
+- `value` — value with any precision. [Int64](../data-types/int-uint.md).
+- `timezone` — (optional) timezone name of the result. [String](../data-types/string.md).
+
+**Returned value**
+
+- `value` converted to DateTime64 with precision `6`. [DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+Query:
+
+``` sql
+WITH CAST(1234567891011, 'Int64') AS i64
+SELECT
+ fromUnixTimestamp64Micro(i64, 'UTC') AS x,
+ toTypeName(x);
+```
+
+Result:
+
+```response
+┌──────────────────────────x─┬─toTypeName(x)────────┐
+│ 1970-01-15 06:56:07.891011 │ DateTime64(6, 'UTC') │
+└────────────────────────────┴──────────────────────┘
+```
+
+## fromUnixTimestamp64Nano
+
+Converts an `Int64` to a `DateTime64` value with fixed nanosecond precision and optional timezone. The input value is scaled up or down appropriately depending on its precision.
+
+:::note
+Please note that input value is treated as a UTC timestamp, not timestamp at the given (or implicit) timezone.
+:::
+
+**Syntax**
+
+``` sql
+fromUnixTimestamp64Nano(value[, timezone])
+```
+
+**Arguments**
+
+- `value` — value with any precision. [Int64](../data-types/int-uint.md).
+- `timezone` — (optional) timezone name of the result. [String](../data-types/string.md).
+
+**Returned value**
+
+- `value` converted to DateTime64 with precision `9`. [DateTime64](../data-types/datetime64.md).
+
+**Example**
+
+Query:
+
+``` sql
+WITH CAST(1234567891011, 'Int64') AS i64
+SELECT
+ fromUnixTimestamp64Nano(i64, 'UTC') AS x,
+ toTypeName(x);
+```
+
+Result:
+
+```response
+┌─────────────────────────────x─┬─toTypeName(x)────────┐
+│ 1970-01-01 00:20:34.567891011 │ DateTime64(9, 'UTC') │
+└───────────────────────────────┴──────────────────────┘
```
## formatRow
diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md
index 8b3e4f44840..76c0141ac8b 100644
--- a/docs/en/sql-reference/functions/url-functions.md
+++ b/docs/en/sql-reference/functions/url-functions.md
@@ -818,6 +818,40 @@ The same as above, but including query string and fragment.
Example: `/top/news.html?page=2#comments`.
+### protocol
+
+Extracts the protocol from a URL.
+
+**Syntax**
+
+```sql
+protocol(url)
+```
+
+**Arguments**
+
+- `url` — URL to extract protocol from. [String](../data-types/string.md).
+
+**Returned value**
+
+- Protocol, or an empty string if it cannot be determined. [String](../data-types/string.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT protocol('https://clickhouse.com/');
+```
+
+Result:
+
+```response
+┌─protocol('https://clickhouse.com/')─┐
+│ https │
+└─────────────────────────────────────┘
+```
+
### queryString
Returns the query string without the initial question mark, `#` and everything after `#`.
diff --git a/docs/en/sql-reference/functions/uuid-functions.md b/docs/en/sql-reference/functions/uuid-functions.md
index 0323ae728a9..e990023efbc 100644
--- a/docs/en/sql-reference/functions/uuid-functions.md
+++ b/docs/en/sql-reference/functions/uuid-functions.md
@@ -150,7 +150,7 @@ The function also works for [Arrays](array-functions.md#function-empty) and [Str
**Example**
-To generate the UUID value, ClickHouse provides the [generateUUIDv4](#uuid-function-generate) function.
+To generate the UUID value, ClickHouse provides the [generateUUIDv4](#generateuuidv4) function.
Query:
@@ -190,7 +190,7 @@ The function also works for [Arrays](array-functions.md#function-notempty) or [S
**Example**
-To generate the UUID value, ClickHouse provides the [generateUUIDv4](#uuid-function-generate) function.
+To generate the UUID value, ClickHouse provides the [generateUUIDv4](#generateuuidv4) function.
Query:
@@ -543,12 +543,17 @@ serverUUID()
Generates a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID).
-The generated Snowflake ID contains the current Unix timestamp in milliseconds 41 (+ 1 top zero bit) bits, followed by machine id (10 bits), a counter (12 bits) to distinguish IDs within a millisecond.
+The generated Snowflake ID contains the current Unix timestamp in milliseconds (41 + 1 top zero bits), followed by a machine id (10 bits), and a counter (12 bits) to distinguish IDs within a millisecond.
For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes.
In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0.
Function `generateSnowflakeID` guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.
+:::note
+The generated Snowflake IDs are based on the UNIX epoch 1970-01-01.
+While no standard or recommendation exists for the epoch of Snowflake IDs, implementations in other systems may use a different epoch, e.g. Twitter/X (2010-11-04) or Mastodon (2015-01-01).
+:::
+
```
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
@@ -605,6 +610,11 @@ SELECT generateSnowflakeID(1), generateSnowflakeID(2);
## snowflakeToDateTime
+:::warning
+This function is deprecated and can only be used if setting [allow_deprecated_snowflake_conversion_functions](../../operations/settings/settings.md#allow_deprecated_snowflake_conversion_functions) is enabled.
+The function will be removed at some point in future.
+:::
+
Extracts the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) in [DateTime](../data-types/datetime.md) format.
**Syntax**
@@ -641,6 +651,11 @@ Result:
## snowflakeToDateTime64
+:::warning
+This function is deprecated and can only be used if setting [allow_deprecated_snowflake_conversion_functions](../../operations/settings/settings.md#allow_deprecated_snowflake_conversion_functions) is enabled.
+The function will be removed at some point in future.
+:::
+
Extracts the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) in [DateTime64](../data-types/datetime64.md) format.
**Syntax**
@@ -677,6 +692,11 @@ Result:
## dateTimeToSnowflake
+:::warning
+This function is deprecated and can only be used if setting [allow_deprecated_snowflake_conversion_functions](../../operations/settings/settings.md#allow_deprecated_snowflake_conversion_functions) is enabled.
+The function will be removed at some point in future.
+:::
+
Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
**Syntax**
@@ -711,6 +731,11 @@ Result:
## dateTime64ToSnowflake
+:::warning
+This function is deprecated and can only be used if setting [allow_deprecated_snowflake_conversion_functions](../../operations/settings/settings.md#allow_deprecated_snowflake_conversion_functions) is enabled.
+The function will be removed at some point in future.
+:::
+
Convert a [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
**Syntax**
@@ -743,6 +768,148 @@ Result:
└─────────────────────────────┘
```
+## snowflakeIDToDateTime
+
+Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime](../data-types/datetime.md).
+
+**Syntax**
+
+``` sql
+snowflakeIDToDateTime(value[, epoch[, time_zone]])
+```
+
+**Arguments**
+
+- `value` — Snowflake ID. [UInt64](../data-types/int-uint.md).
+- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
+- `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md).
+
+**Returned value**
+
+- The timestamp component of `value` as a [DateTime](../data-types/datetime.md) value.
+
+**Example**
+
+Query:
+
+```sql
+SELECT snowflakeIDToDateTime(7204436857747984384) AS res
+```
+
+Result:
+
+```
+┌─────────────────res─┐
+│ 2024-06-06 10:59:58 │
+└─────────────────────┘
+```
+
+## snowflakeIDToDateTime64
+
+Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime64](../data-types/datetime64.md).
+
+**Syntax**
+
+``` sql
+snowflakeIDToDateTime64(value[, epoch[, time_zone]])
+```
+
+**Arguments**
+
+- `value` — Snowflake ID. [UInt64](../data-types/int-uint.md).
+- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
+- `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md).
+
+**Returned value**
+
+- The timestamp component of `value` as a [DateTime64](../data-types/datetime64.md) with scale = 3, i.e. millisecond precision.
+
+**Example**
+
+Query:
+
+```sql
+SELECT snowflakeIDToDateTime64(7204436857747984384) AS res
+```
+
+Result:
+
+```
+┌─────────────────res─┐
+│ 2024-06-06 10:59:58 │
+└─────────────────────┘
+```
+
+## dateTimeToSnowflakeID
+
+Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
+
+**Syntax**
+
+``` sql
+dateTimeToSnowflakeID(value[, epoch])
+```
+
+**Arguments**
+
+- `value` — Date with time. [DateTime](../data-types/datetime.md).
+- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
+
+**Returned value**
+
+- Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.
+
+**Example**
+
+Query:
+
+```sql
+SELECT toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt, dateTimeToSnowflakeID(dt) AS res;
+```
+
+Result:
+
+```
+┌──────────────────dt─┬─────────────────res─┐
+│ 2021-08-15 18:57:56 │ 6832626392367104000 │
+└─────────────────────┴─────────────────────┘
+```
+
+## dateTime64ToSnowflakeID
+
+Convert a [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
+
+**Syntax**
+
+``` sql
+dateTime64ToSnowflakeID(value[, epoch])
+```
+
+**Arguments**
+
+- `value` — Date with time. [DateTime64](../data-types/datetime64.md).
+- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
+
+**Returned value**
+
+- Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.
+
+**Example**
+
+Query:
+
+```sql
+SELECT toDateTime('2021-08-15 18:57:56.493', 3, 'Asia/Shanghai') AS dt, dateTime64ToSnowflakeID(dt) AS res;
+```
+
+Result:
+
+```
+┌──────────────────────dt─┬─────────────────res─┐
+│ 2021-08-15 18:57:56.493 │ 6832626394434895872 │
+└─────────────────────────┴─────────────────────┘
+```
+
## See also
- [dictGetUUID](../functions/ext-dict-functions.md#ext_dict_functions-other)
diff --git a/docs/en/sql-reference/operators/in.md b/docs/en/sql-reference/operators/in.md
index 0257d21b30f..ed75b1802d8 100644
--- a/docs/en/sql-reference/operators/in.md
+++ b/docs/en/sql-reference/operators/in.md
@@ -235,7 +235,7 @@ If `some_predicate` is not selective enough, it will return a large amount of da
### Distributed Subqueries and max_parallel_replicas
-When [max_parallel_replicas](#settings-max_parallel_replicas) is greater than 1, distributed queries are further transformed.
+When [max_parallel_replicas](#distributed-subqueries-and-max_parallel_replicas) is greater than 1, distributed queries are further transformed.
For example, the following:
@@ -255,7 +255,7 @@ where `M` is between `1` and `3` depending on which replica the local query is e
These settings affect every MergeTree-family table in the query and have the same effect as applying `SAMPLE 1/3 OFFSET (M-1)/3` on each table.
-Therefore adding the [max_parallel_replicas](#settings-max_parallel_replicas) setting will only produce correct results if both tables have the same replication scheme and are sampled by UserID or a subkey of it. In particular, if `local_table_2` does not have a sampling key, incorrect results will be produced. The same rule applies to `JOIN`.
+Therefore adding the [max_parallel_replicas](#distributed-subqueries-and-max_parallel_replicas) setting will only produce correct results if both tables have the same replication scheme and are sampled by UserID or a subkey of it. In particular, if `local_table_2` does not have a sampling key, incorrect results will be produced. The same rule applies to `JOIN`.
One workaround if `local_table_2` does not meet the requirements, is to use `GLOBAL IN` or `GLOBAL JOIN`.
diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md
index a23710b12bd..aa6f132e08e 100644
--- a/docs/en/sql-reference/statements/alter/column.md
+++ b/docs/en/sql-reference/statements/alter/column.md
@@ -108,7 +108,7 @@ ALTER TABLE visits RENAME COLUMN webBrowser TO browser
CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name
```
-Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to set the partition expression](partition.md/#how-to-set-partition-expression).
+Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to set the partition expression](../alter/partition.md/#how-to-set-partition-expression).
If the `IF EXISTS` clause is specified, the query won’t return an error if the column does not exist.
@@ -173,7 +173,7 @@ ALTER TABLE visits MODIFY COLUMN browser Array(String)
Changing the column type is the only complex action – it changes the contents of files with data. For large tables, this may take a long time.
-The query also can change the order of the columns using `FIRST | AFTER` clause, see [ADD COLUMN](#alter_add-column) description, but column type is mandatory in this case.
+The query also can change the order of the columns using `FIRST | AFTER` clause, see [ADD COLUMN](#add-column) description, but column type is mandatory in this case.
Example:
diff --git a/docs/en/sql-reference/statements/alter/constraint.md b/docs/en/sql-reference/statements/alter/constraint.md
index 29675f704b5..54c456f9aa2 100644
--- a/docs/en/sql-reference/statements/alter/constraint.md
+++ b/docs/en/sql-reference/statements/alter/constraint.md
@@ -9,8 +9,8 @@ sidebar_label: CONSTRAINT
Constraints could be added or deleted using following syntax:
``` sql
-ALTER TABLE [db].name [ON CLUSTER cluster] ADD CONSTRAINT constraint_name CHECK expression;
-ALTER TABLE [db].name [ON CLUSTER cluster] DROP CONSTRAINT constraint_name;
+ALTER TABLE [db].name [ON CLUSTER cluster] ADD CONSTRAINT [IF NOT EXISTS] constraint_name CHECK expression;
+ALTER TABLE [db].name [ON CLUSTER cluster] DROP CONSTRAINT [IF EXISTS] constraint_name;
```
See more on [constraints](../../../sql-reference/statements/create/table.md#constraints).
diff --git a/docs/en/sql-reference/statements/alter/index.md b/docs/en/sql-reference/statements/alter/index.md
index 3cfb99cff83..f81d4f02e0c 100644
--- a/docs/en/sql-reference/statements/alter/index.md
+++ b/docs/en/sql-reference/statements/alter/index.md
@@ -16,7 +16,7 @@ Most `ALTER TABLE` queries modify table settings or data:
- [INDEX](/docs/en/sql-reference/statements/alter/skipping-index.md)
- [CONSTRAINT](/docs/en/sql-reference/statements/alter/constraint.md)
- [TTL](/docs/en/sql-reference/statements/alter/ttl.md)
-- [STATISTIC](/docs/en/sql-reference/statements/alter/statistic.md)
+- [STATISTICS](/docs/en/sql-reference/statements/alter/statistics.md)
- [APPLY DELETED MASK](/docs/en/sql-reference/statements/alter/apply-deleted-mask.md)
:::note
diff --git a/docs/en/sql-reference/statements/alter/named-collection.md b/docs/en/sql-reference/statements/alter/named-collection.md
index 71d4bfadd9c..ab772fe4dcf 100644
--- a/docs/en/sql-reference/statements/alter/named-collection.md
+++ b/docs/en/sql-reference/statements/alter/named-collection.md
@@ -3,6 +3,10 @@ slug: /en/sql-reference/statements/alter/named-collection
sidebar_label: NAMED COLLECTION
---
+import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
+
+
+
# ALTER NAMED COLLECTION
This query intends to modify already existing named collections.
diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md
index ce5cecf6fd6..778816f8934 100644
--- a/docs/en/sql-reference/statements/alter/partition.md
+++ b/docs/en/sql-reference/statements/alter/partition.md
@@ -31,7 +31,7 @@ The following operations with [partitions](/docs/en/engines/table-engines/merget
ALTER TABLE table_name [ON CLUSTER cluster] DETACH PARTITION|PART partition_expr
```
-Moves all data for the specified partition to the `detached` directory. The server forgets about the detached data partition as if it does not exist. The server will not know about this data until you make the [ATTACH](#alter_attach-partition) query.
+Moves all data for the specified partition to the `detached` directory. The server forgets about the detached data partition as if it does not exist. The server will not know about this data until you make the [ATTACH](#attach-partitionpart) query.
Example:
@@ -139,7 +139,7 @@ For the query to run successfully, the following conditions must be met:
ALTER TABLE table2 [ON CLUSTER cluster] REPLACE PARTITION partition_expr FROM table1
```
-This query copies the data partition from the `table1` to `table2` and replaces existing partition in the `table2`.
+This query copies the data partition from `table1` to `table2` and replaces the existing partition in `table2`. The operation is atomic.
Note that:
@@ -252,7 +252,7 @@ Downloads a partition from another server. This query only works for the replica
The query does the following:
1. Downloads the partition|part from the specified shard. In ‘path-in-zookeeper’ you must specify a path to the shard in ZooKeeper.
-2. Then the query puts the downloaded data to the `detached` directory of the `table_name` table. Use the [ATTACH PARTITION\|PART](#alter_attach-partition) query to add the data to the table.
+2. Then the query puts the downloaded data to the `detached` directory of the `table_name` table. Use the [ATTACH PARTITION\|PART](#attach-partitionpart) query to add the data to the table.
For example:
@@ -353,7 +353,7 @@ You can specify the partition expression in `ALTER ... PARTITION` queries in dif
- Using the keyword `ALL`. It can be used only with DROP/DETACH/ATTACH. For example, `ALTER TABLE visits ATTACH PARTITION ALL`.
- As a tuple of expressions or constants that matches (in types) the table partitioning keys tuple. In the case of a single element partitioning key, the expression should be wrapped in the `tuple (...)` function. For example, `ALTER TABLE visits DETACH PARTITION tuple(toYYYYMM(toDate('2019-01-25')))`.
- Using the partition ID. Partition ID is a string identifier of the partition (human-readable, if possible) that is used as the names of partitions in the file system and in ZooKeeper. The partition ID must be specified in the `PARTITION ID` clause, in a single quotes. For example, `ALTER TABLE visits DETACH PARTITION ID '201901'`.
-- In the [ALTER ATTACH PART](#alter_attach-partition) and [DROP DETACHED PART](#alter_drop-detached) query, to specify the name of a part, use string literal with a value from the `name` column of the [system.detached_parts](/docs/en/operations/system-tables/detached_parts.md/#system_tables-detached_parts) table. For example, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`.
+- In the [ALTER ATTACH PART](#attach-partitionpart) and [DROP DETACHED PART](#drop-detached-partitionpart) query, to specify the name of a part, use string literal with a value from the `name` column of the [system.detached_parts](/docs/en/operations/system-tables/detached_parts.md/#system_tables-detached_parts) table. For example, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`.
Usage of quotes when specifying the partition depends on the type of partition expression. For example, for the `String` type, you have to specify its name in quotes (`'`). For the `Date` and `Int*` types no quotes are needed.
diff --git a/docs/en/sql-reference/statements/alter/statistic.md b/docs/en/sql-reference/statements/alter/statistic.md
deleted file mode 100644
index 1c2e45b23fd..00000000000
--- a/docs/en/sql-reference/statements/alter/statistic.md
+++ /dev/null
@@ -1,25 +0,0 @@
----
-slug: /en/sql-reference/statements/alter/statistic
-sidebar_position: 45
-sidebar_label: STATISTIC
----
-
-# Manipulating Column Statistics
-
-The following operations are available:
-
-- `ALTER TABLE [db].table ADD STATISTIC (columns list) TYPE type` - Adds statistic description to tables metadata.
-
-- `ALTER TABLE [db].table DROP STATISTIC (columns list) TYPE type` - Removes statistic description from tables metadata and deletes statistic files from disk.
-
-- `ALTER TABLE [db].table CLEAR STATISTIC (columns list) TYPE type` - Deletes statistic files from disk.
-
-- `ALTER TABLE [db.]table MATERIALIZE STATISTIC (columns list) TYPE type` - Rebuilds the statistic for columns. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
-
-The first two commands are lightweight in a sense that they only change metadata or remove files.
-
-Also, they are replicated, syncing statistics metadata via ZooKeeper.
-
-:::note
-Statistic manipulation is supported only for tables with [`*MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) engine (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) variants).
-:::
diff --git a/docs/en/sql-reference/statements/alter/statistics.md b/docs/en/sql-reference/statements/alter/statistics.md
new file mode 100644
index 00000000000..80024781f88
--- /dev/null
+++ b/docs/en/sql-reference/statements/alter/statistics.md
@@ -0,0 +1,33 @@
+---
+slug: /en/sql-reference/statements/alter/statistics
+sidebar_position: 45
+sidebar_label: STATISTICS
+---
+
+# Manipulating Column Statistics
+
+The following operations are available:
+
+- `ALTER TABLE [db].table ADD STATISTICS (columns list) TYPE (type list)` - Adds statistic description to tables metadata.
+
+- `ALTER TABLE [db].table MODIFY STATISTICS (columns list) TYPE (type list)` - Modifies statistic description to tables metadata.
+
+- `ALTER TABLE [db].table DROP STATISTICS (columns list)` - Removes statistics from the metadata of the specified columns and deletes all statistics objects in all parts for the specified columns.
+
+- `ALTER TABLE [db].table CLEAR STATISTICS (columns list)` - Deletes all statistics objects in all parts for the specified columns. Statistics objects can be rebuild using `ALTER TABLE MATERIALIZE STATISTICS`.
+
+- `ALTER TABLE [db.]table MATERIALIZE STATISTICS (columns list)` - Rebuilds the statistic for columns. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
+
+The first two commands are lightweight in a sense that they only change metadata or remove files.
+
+Also, they are replicated, syncing statistics metadata via ZooKeeper.
+
+There is an example adding two statistics types to two columns:
+
+```
+ALTER TABLE t1 MODIFY STATISTICS c, d TYPE TDigest, Uniq;
+```
+
+:::note
+Statistic manipulation is supported only for tables with [`*MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) engine (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) variants).
+:::
diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md
index b5c156f56a9..6216b83c2ef 100644
--- a/docs/en/sql-reference/statements/alter/user.md
+++ b/docs/en/sql-reference/statements/alter/user.md
@@ -12,7 +12,7 @@ Syntax:
``` sql
ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
- [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'}]
+ [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'}]
[[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
[VALID UNTIL datetime]
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
diff --git a/docs/en/sql-reference/statements/alter/view.md b/docs/en/sql-reference/statements/alter/view.md
index fb7a5bd7c03..5f3dae0a9c0 100644
--- a/docs/en/sql-reference/statements/alter/view.md
+++ b/docs/en/sql-reference/statements/alter/view.md
@@ -134,8 +134,8 @@ PRIMARY KEY (event_type, ts)
ORDER BY (event_type, ts, browser)
SETTINGS index_granularity = 8192
--- !!! The columns' definition is unchanged but it does not matter, we are not quering
--- MATERIALIZED VIEW, we are quering TO (storage) table.
+-- !!! The columns' definition is unchanged but it does not matter, we are not querying
+-- MATERIALIZED VIEW, we are querying TO (storage) table.
-- SELECT section is updated.
SHOW CREATE TABLE mv FORMAT TSVRaw;
diff --git a/docs/en/sql-reference/statements/create/named-collection.md b/docs/en/sql-reference/statements/create/named-collection.md
index f69fa2e3678..a4e146c814c 100644
--- a/docs/en/sql-reference/statements/create/named-collection.md
+++ b/docs/en/sql-reference/statements/create/named-collection.md
@@ -3,6 +3,10 @@ slug: /en/sql-reference/statements/create/named-collection
sidebar_label: NAMED COLLECTION
---
+import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
+
+
+
# CREATE NAMED COLLECTION
Creates a new named collection.
diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md
index 628fe1d2875..0253bc647e6 100644
--- a/docs/en/sql-reference/statements/create/table.md
+++ b/docs/en/sql-reference/statements/create/table.md
@@ -17,8 +17,8 @@ By default, tables are created only on the current server. Distributed DDL queri
``` sql
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
(
- name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr1] [compression_codec] [TTL expr1] [COMMENT 'comment for column'],
- name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr2] [compression_codec] [TTL expr2] [COMMENT 'comment for column'],
+ name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr1] [COMMENT 'comment for column'] [compression_codec] [TTL expr1],
+ name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr2] [COMMENT 'comment for column'] [compression_codec] [TTL expr2],
...
) ENGINE = engine
COMMENT 'comment for table'
diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md
index aee98cfcd10..8c9143ee086 100644
--- a/docs/en/sql-reference/statements/create/user.md
+++ b/docs/en/sql-reference/statements/create/user.md
@@ -12,7 +12,7 @@ Syntax:
``` sql
CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
[, name2 [ON CLUSTER cluster_name2] ...]
- [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}]
+ [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}]
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
[VALID UNTIL datetime]
[IN access_storage_type]
diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md
index 1bdf22b35b0..1fabb6d8cc7 100644
--- a/docs/en/sql-reference/statements/create/view.md
+++ b/docs/en/sql-reference/statements/create/view.md
@@ -6,7 +6,7 @@ sidebar_label: VIEW
# CREATE VIEW
-Creates a new view. Views can be [normal](#normal-view), [materialized](#materialized-view), [live](#live-view-experimental), and [window](#window-view-experimental) (live view and window view are experimental features).
+Creates a new view. Views can be [normal](#normal-view), [materialized](#materialized-view), [live](#live-view-deprecated), and [window](#window-view-experimental) (live view and window view are experimental features).
## Normal View
diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md
index 2850ce71781..43fa344a16d 100644
--- a/docs/en/sql-reference/statements/grant.md
+++ b/docs/en/sql-reference/statements/grant.md
@@ -33,7 +33,7 @@ GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_US
- `role` — ClickHouse user role.
- `user` — ClickHouse user account.
-The `WITH ADMIN OPTION` clause grants [ADMIN OPTION](#admin-option-privilege) privilege to `user` or `role`.
+The `WITH ADMIN OPTION` clause grants [ADMIN OPTION](#admin-option) privilege to `user` or `role`.
The `WITH REPLACE OPTION` clause replace old roles by new role for the `user` or `role`, if is not specified it appends roles.
## Grant Current Grants Syntax
@@ -201,7 +201,7 @@ Hierarchy of privileges:
- `HDFS`
- `S3`
- [dictGet](#dictget)
-- [displaySecretsInShowAndSelect](#display-secrets)
+- [displaySecretsInShowAndSelect](#displaysecretsinshowandselect)
- [NAMED COLLECTION ADMIN](#named-collection-admin)
- `CREATE NAMED COLLECTION`
- `DROP NAMED COLLECTION`
@@ -498,7 +498,7 @@ Privilege level: `DICTIONARY`.
- `GRANT dictGet ON mydictionary TO john`
-### displaySecretsInShowAndSelect {#display-secrets}
+### displaySecretsInShowAndSelect
Allows a user to view secrets in `SHOW` and `SELECT` queries if both
[`display_secrets_in_show_and_select` server setting](../../operations/server-configuration-parameters/settings#display_secrets_in_show_and_select)
diff --git a/docs/en/sql-reference/statements/select/sample.md b/docs/en/sql-reference/statements/select/sample.md
index 137f86cc8b9..78e05b19bd1 100644
--- a/docs/en/sql-reference/statements/select/sample.md
+++ b/docs/en/sql-reference/statements/select/sample.md
@@ -27,14 +27,14 @@ The features of data sampling are listed below:
For the `SAMPLE` clause the following syntax is supported:
-| SAMPLE Clause Syntax | Description |
-|----------------------|------------------------------|
-| `SAMPLE k` | Here `k` is the number from 0 to 1. The query is executed on `k` fraction of data. For example, `SAMPLE 0.1` runs the query on 10% of data. [Read more](#select-sample-k) |
-| `SAMPLE n` | Here `n` is a sufficiently large integer. The query is executed on a sample of at least `n` rows (but not significantly more than this). For example, `SAMPLE 10000000` runs the query on a minimum of 10,000,000 rows. [Read more](#select-sample-n) |
-| `SAMPLE k OFFSET m` | Here `k` and `m` are the numbers from 0 to 1. The query is executed on a sample of `k` fraction of the data. The data used for the sample is offset by `m` fraction. [Read more](#select-sample-offset) |
+| SAMPLE Clause Syntax | Description |
+|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `SAMPLE k` | Here `k` is the number from 0 to 1. The query is executed on `k` fraction of data. For example, `SAMPLE 0.1` runs the query on 10% of data. [Read more](#sample-k) |
+| `SAMPLE n` | Here `n` is a sufficiently large integer. The query is executed on a sample of at least `n` rows (but not significantly more than this). For example, `SAMPLE 10000000` runs the query on a minimum of 10,000,000 rows. [Read more](#sample-n) |
+| `SAMPLE k OFFSET m` | Here `k` and `m` are the numbers from 0 to 1. The query is executed on a sample of `k` fraction of the data. The data used for the sample is offset by `m` fraction. [Read more](#sample-k-offset-m) |
-## SAMPLE K {#select-sample-k}
+## SAMPLE K
Here `k` is the number from 0 to 1 (both fractional and decimal notations are supported). For example, `SAMPLE 1/2` or `SAMPLE 0.5`.
@@ -54,7 +54,7 @@ ORDER BY PageViews DESC LIMIT 1000
In this example, the query is executed on a sample from 0.1 (10%) of data. Values of aggregate functions are not corrected automatically, so to get an approximate result, the value `count()` is manually multiplied by 10.
-## SAMPLE N {#select-sample-n}
+## SAMPLE N
Here `n` is a sufficiently large integer. For example, `SAMPLE 10000000`.
@@ -90,7 +90,7 @@ FROM visits
SAMPLE 10000000
```
-## SAMPLE K OFFSET M {#select-sample-offset}
+## SAMPLE K OFFSET M
Here `k` and `m` are numbers from 0 to 1. Examples are shown below.
diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md
index 7efbff1b42b..e6d3439d2b9 100644
--- a/docs/en/sql-reference/statements/system.md
+++ b/docs/en/sql-reference/statements/system.md
@@ -174,7 +174,7 @@ Aborts ClickHouse process (like `kill -9 {$ pid_clickhouse-server}`)
## Managing Distributed Tables
-ClickHouse can manage [distributed](../../engines/table-engines/special/distributed.md) tables. When a user inserts data into these tables, ClickHouse first creates a queue of the data that should be sent to cluster nodes, then asynchronously sends it. You can manage queue processing with the [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends), [FLUSH DISTRIBUTED](#query_language-system-flush-distributed), and [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) queries. You can also synchronously insert distributed data with the [distributed_foreground_insert](../../operations/settings/settings.md#distributed_foreground_insert) setting.
+ClickHouse can manage [distributed](../../engines/table-engines/special/distributed.md) tables. When a user inserts data into these tables, ClickHouse first creates a queue of the data that should be sent to cluster nodes, then asynchronously sends it. You can manage queue processing with the [STOP DISTRIBUTED SENDS](#stop-distributed-sends), [FLUSH DISTRIBUTED](#flush-distributed), and [START DISTRIBUTED SENDS](#start-distributed-sends) queries. You can also synchronously insert distributed data with the [distributed_foreground_insert](../../operations/settings/settings.md#distributed_foreground_insert) setting.
### STOP DISTRIBUTED SENDS
diff --git a/docs/en/sql-reference/syntax.md b/docs/en/sql-reference/syntax.md
index fc0286e76ad..6a4afb63db8 100644
--- a/docs/en/sql-reference/syntax.md
+++ b/docs/en/sql-reference/syntax.md
@@ -54,11 +54,11 @@ Identifiers are:
- Cluster, database, table, partition, and column names.
- Functions.
- Data types.
-- [Expression aliases](#expression_aliases).
+- [Expression aliases](#expression-aliases).
Identifiers can be quoted or non-quoted. The latter is preferred.
-Non-quoted identifiers must match the regex `^[a-zA-Z_][0-9a-zA-Z_]*$` and can not be equal to [keywords](#syntax-keywords). Examples: `x`, `_1`, `X_y__Z123_`.
+Non-quoted identifiers must match the regex `^[a-zA-Z_][0-9a-zA-Z_]*$` and can not be equal to [keywords](#keywords). Examples: `x`, `_1`, `X_y__Z123_`.
If you want to use identifiers the same as keywords or you want to use other symbols in identifiers, quote it using double quotes or backticks, for example, `"id"`, `` `id` ``.
diff --git a/docs/en/sql-reference/table-functions/azureBlobStorage.md b/docs/en/sql-reference/table-functions/azureBlobStorage.md
index 1510489ce83..f59fedeb3a2 100644
--- a/docs/en/sql-reference/table-functions/azureBlobStorage.md
+++ b/docs/en/sql-reference/table-functions/azureBlobStorage.md
@@ -72,6 +72,7 @@ SELECT count(*) FROM azureBlobStorage('DefaultEndpointsProtocol=https;AccountNam
- `_path` — Path to the file. Type: `LowCardinalty(String)`.
- `_file` — Name of the file. Type: `LowCardinalty(String)`.
- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the file size is unknown, the value is `NULL`.
+- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
**See Also**
diff --git a/docs/en/sql-reference/table-functions/file.md b/docs/en/sql-reference/table-functions/file.md
index f66178afbb2..3a3162dad9a 100644
--- a/docs/en/sql-reference/table-functions/file.md
+++ b/docs/en/sql-reference/table-functions/file.md
@@ -18,7 +18,7 @@ file([path_to_archive ::] path [,format] [,structure] [,compression])
**Parameters**
-- `path` — The relative path to the file from [user_files_path](/docs/en/operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Supports in read-only mode the following [globs](#globs_in_path): `*`, `?`, `{abc,def}` (with `'abc'` and `'def'` being strings) and `{N..M}` (with `N` and `M` being numbers).
+- `path` — The relative path to the file from [user_files_path](/docs/en/operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Supports in read-only mode the following [globs](#globs-in-path): `*`, `?`, `{abc,def}` (with `'abc'` and `'def'` being strings) and `{N..M}` (with `N` and `M` being numbers).
- `path_to_archive` - The relative path to a zip/tar/7z archive. Supports the same globs as `path`.
- `format` — The [format](/docs/en/interfaces/formats.md#formats) of the file.
- `structure` — Structure of the table. Format: `'column1_name column1_type, column2_name column2_type, ...'`.
@@ -128,7 +128,7 @@ Reading data from `table.csv`, located in `archive1.zip` or/and `archive2.zip`:
SELECT * FROM file('user_files/archives/archive{1..2}.zip :: table.csv');
```
-## Globs in path {#globs_in_path}
+## Globs in path
Paths may use globbing. Files must match the whole path pattern, not only the suffix or prefix.
@@ -196,6 +196,7 @@ SELECT count(*) FROM file('big_dir/**/file002', 'CSV', 'name String, value UInt3
- `_path` — Path to the file. Type: `LowCardinalty(String)`.
- `_file` — Name of the file. Type: `LowCardinalty(String)`.
- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the file size is unknown, the value is `NULL`.
+- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
## Settings {#settings}
diff --git a/docs/en/sql-reference/table-functions/fileCluster.md b/docs/en/sql-reference/table-functions/fileCluster.md
index 4677d2883a7..62b00fadd62 100644
--- a/docs/en/sql-reference/table-functions/fileCluster.md
+++ b/docs/en/sql-reference/table-functions/fileCluster.md
@@ -22,7 +22,7 @@ fileCluster(cluster_name, path[, format, structure, compression_method])
**Arguments**
- `cluster_name` — Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers.
-- `path` — The relative path to the file from [user_files_path](/docs/en/operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Path to file also supports [globs](#globs_in_path).
+- `path` — The relative path to the file from [user_files_path](/docs/en/operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Path to file also supports [globs](#globs-in-path).
- `format` — [Format](../../interfaces/formats.md#formats) of the files. Type: [String](../../sql-reference/data-types/string.md).
- `structure` — Table structure in `'UserID UInt64, Name String'` format. Determines column names and types. Type: [String](../../sql-reference/data-types/string.md).
- `compression_method` — Compression method. Supported compression types are `gz`, `br`, `xz`, `zst`, `lz4`, and `bz2`.
@@ -74,7 +74,7 @@ SELECT * FROM fileCluster('my_cluster', 'file{1,2}.csv', 'CSV', 'i UInt32, s Str
```
-## Globs in Path {#globs_in_path}
+## Globs in Path
All patterns supported by [File](../../sql-reference/table-functions/file.md#globs-in-path) table function are supported by FileCluster.
diff --git a/docs/en/sql-reference/table-functions/hdfs.md b/docs/en/sql-reference/table-functions/hdfs.md
index d65615e7588..28cba5ccc6a 100644
--- a/docs/en/sql-reference/table-functions/hdfs.md
+++ b/docs/en/sql-reference/table-functions/hdfs.md
@@ -97,6 +97,7 @@ FROM hdfs('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name Strin
- `_path` — Path to the file. Type: `LowCardinalty(String)`.
- `_file` — Name of the file. Type: `LowCardinalty(String)`.
- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the size is unknown, the value is `NULL`.
+- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
## Storage Settings {#storage-settings}
diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md
index cbef80371a3..35e5d86034c 100644
--- a/docs/en/sql-reference/table-functions/s3.md
+++ b/docs/en/sql-reference/table-functions/s3.md
@@ -269,9 +269,10 @@ FROM s3(
## Virtual Columns {#virtual-columns}
-- `_path` — Path to the file. Type: `LowCardinalty(String)`.
-- `_file` — Name of the file. Type: `LowCardinalty(String)`.
-- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the file size is unknown, the value is `NULL`.
+- `_path` — Path to the file. Type: `LowCardinalty(String)`. In case of archive, shows path in a format: "{path_to_archive}::{path_to_file_inside_archive}"
+- `_file` — Name of the file. Type: `LowCardinalty(String)`. In case of archive shows name of the file inside the archive.
+- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the file size is unknown, the value is `NULL`. In case of archive shows uncompressed file size of the file inside the archive.
+- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
## Storage Settings {#storage-settings}
diff --git a/docs/en/sql-reference/table-functions/url.md b/docs/en/sql-reference/table-functions/url.md
index 4dc6e435b50..3bb7aff53a7 100644
--- a/docs/en/sql-reference/table-functions/url.md
+++ b/docs/en/sql-reference/table-functions/url.md
@@ -53,6 +53,7 @@ Character `|` inside patterns is used to specify failover addresses. They are it
- `_path` — Path to the `URL`. Type: `LowCardinalty(String)`.
- `_file` — Resource name of the `URL`. Type: `LowCardinalty(String)`.
- `_size` — Size of the resource in bytes. Type: `Nullable(UInt64)`. If the size is unknown, the value is `NULL`.
+- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`.
## Storage Settings {#storage-settings}
diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md
index 32ebc6d028f..49076f3cbe1 100644
--- a/docs/en/sql-reference/window-functions/index.md
+++ b/docs/en/sql-reference/window-functions/index.md
@@ -36,7 +36,7 @@ Finds non-negative derivative for given `metric_column` by `timestamp_column`.
`INTERVAL` can be omitted, default is `INTERVAL 1 SECOND`.
The computed value is the following for each row:
- `0` for 1st row,
-- ${metric_i - metric_{i-1} \over timestamp_i - timestamp_{i-1}} * interval$ for $i_th$ row.
+- ${\text{metric}_i - \text{metric}_{i-1} \over \text{timestamp}_i - \text{timestamp}_{i-1}} * \text{interval}$ for $i_{th}$ row.
## Syntax
@@ -80,8 +80,8 @@ These functions can be used only as a window function.
- `nth_value(x, offset)` - Return the first non-NULL value evaluated against the nth row (offset) in its ordered frame.
- `rank()` - Rank the current row within its partition with gaps.
- `dense_rank()` - Rank the current row within its partition without gaps.
-- `lagInFrame(x)` - Return a value evaluated at the row that is at a specified physical offset row before the current row within the ordered frame.
-- `leadInFrame(x)` - Return a value evaluated at the row that is offset rows after the current row within the ordered frame.
+- `lagInFrame(x[, offset[, default]])` - Return a value evaluated at the row that is at a specified physical offset row before the current row within the ordered frame. The offset parameter, if not specified, defaults to 1, meaning it will fetch the value from the next row. If the calculated row exceeds the boundaries of the window frame, the specified default value is returned.
+- `leadInFrame(x[, offset[, default]])` - Return a value evaluated at the row that is offset rows after the current row within the ordered frame. If offset is not provided, it defaults to 1. If the offset leads to a position outside the window frame, the specified default value is used.
## Examples
diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md
index 01ff4dd5f28..bf42edf89ff 100644
--- a/docs/ru/development/developer-instruction.md
+++ b/docs/ru/development/developer-instruction.md
@@ -283,7 +283,7 @@ Pull request можно создать, даже если работа над з
Тесты будут запущены, как только сотрудники ClickHouse поставят для pull request тег «Can be tested». Результаты первых проверок (стиль кода) появятся уже через несколько минут. Результаты сборки появятся примерно через пол часа. Результаты основного набора тестов будут доступны в пределах часа.
-Система подготовит сборки ClickHouse специально для вашего pull request. Для их получения, нажмите на ссылку «Details» у проверки «Clickhouse build check». Там вы сможете найти прямые ссылки на собранные .deb пакеты ClickHouse, которые, при желании, вы даже сможете установить на свои продакшен серверы (если не страшно).
+Система подготовит сборки ClickHouse специально для вашего pull request. Для их получения, нажмите на ссылку «Details» у проверки «Builds». Там вы сможете найти прямые ссылки на собранные .deb пакеты ClickHouse, которые, при желании, вы даже сможете установить на свои продакшен серверы (если не страшно).
Вероятнее всего, часть сборок не будет успешной с первого раза. Ведь мы проверяем сборку кода и gcc и clang, а при сборке с помощью clang включаются почти все существующие в природе warnings (всегда с флагом `-Werror`). На той же странице, вы сможете найти логи сборки - вам не обязательно самому собирать ClickHouse всеми возможными способами.
diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md
index 4d19cf50ae1..86eeaac2da7 100644
--- a/docs/ru/interfaces/cli.md
+++ b/docs/ru/interfaces/cli.md
@@ -141,6 +141,7 @@ $ clickhouse-client --param_tbl="numbers" --param_db="system" --param_col="numbe
- `--secure` — если указано, будет использован безопасный канал.
- `--history_file` - путь к файлу с историей команд.
- `--param_` — значение параметра для [запроса с параметрами](#cli-queries-with-parameters).
+- `--jwt` – авторизация с использованием JSON Web Token. Доступно только в ClickHouse Cloud.
Вместо параметров `--host`, `--port`, `--user` и `--password` клиент ClickHouse также поддерживает строки подключения (смотри следующий раздел).
diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md
index 5f11f1b430b..d9da51892f9 100644
--- a/docs/ru/interfaces/http.md
+++ b/docs/ru/interfaces/http.md
@@ -414,6 +414,8 @@ $ curl -v 'http://localhost:8123/predefined_query'
- `content_type` — используется со всеми типами, возвращает [content-type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type).
+ - `http_response_headers` — используется со всеми типами чтобы добавить кастомные хедеры в ответ. Может использоваться в том числе для задания хедера `Content-Type` вместо `content_type`.
+
- `response_content` — используется с типом`static`, содержимое ответа, отправленное клиенту, при использовании префикса ‘file://’ or ‘config://’, находит содержимое из файла или конфигурации, отправленного клиенту.
Далее приведены методы настройки для различных типов.
@@ -509,6 +511,33 @@ max_final_threads 2
static
402
text/html; charset=UTF-8
+
+ en
+ 43
+
+ Say Hi!
+
+
+
+
+```
+
+`http_response_headers` так же может использоваться для определения `Content-Type` вместо `content_type`.
+
+``` xml
+
+
+ GET
+ xxx
+ /hi
+
+ static
+ 402
+
+ text/html; charset=UTF-8
+ en
+ 43
+
Say Hi!
@@ -589,6 +618,9 @@ $ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler'
static
text/html; charset=UTF-8
+
+ 737060cd8c284d8af7ad3082f209582d
+
file:///absolute_path_file.html
@@ -599,6 +631,9 @@ $ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler'
static
text/html; charset=UTF-8
+
+ 737060cd8c284d8af7ad3082f209582d
+
file://./relative_path_file.html
diff --git a/docs/ru/operations/external-authenticators/ssl-x509.md b/docs/ru/operations/external-authenticators/ssl-x509.md
index affdf87b199..7f1fb03962c 100644
--- a/docs/ru/operations/external-authenticators/ssl-x509.md
+++ b/docs/ru/operations/external-authenticators/ssl-x509.md
@@ -3,23 +3,30 @@ slug: /ru/operations/external-authenticators/ssl-x509
---
# Аутентификация по сертификату SSL X.509 {#ssl-external-authentication}
-[Опция 'strict'](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) включает обязательную проверку сертификатов входящих соединений в библиотеке `SSL`. В этом случае могут быть установлены только соединения, представившие действительный сертификат. Соединения с недоверенными сертификатами будут отвергнуты. Таким образом, проверка сертификата позволяет однозначно аутентифицировать входящее соединение. Идентификация пользователя осуществляется по полю `Common Name` сертификата. Это позволяет ассоциировать несколько сертификатов с одним и тем же пользователем. Дополнительно, перевыпуск и отзыв сертификата не требуют изменения конфигурации ClickHouse.
+[Опция 'strict'](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) включает обязательную проверку сертификатов входящих соединений в библиотеке `SSL`. В этом случае могут быть установлены только соединения, представившие действительный сертификат. Соединения с недоверенными сертификатами будут отвергнуты. Таким образом, проверка сертификата позволяет однозначно аутентифицировать входящее соединение. Идентификация пользователя осуществляется по полю `Common Name` или `subjectAltName` сертификата. Это позволяет ассоциировать несколько сертификатов с одним и тем же пользователем. Дополнительно, перевыпуск и отзыв сертификата не требуют изменения конфигурации ClickHouse.
-Для включения аутентификации по SSL сертификату, необходимо указать список `Common Name` для каждого пользователя ClickHouse в файле настройки `config.xml`:
+Для включения аутентификации по SSL сертификату, необходимо указать список `Common Name` или `subjectAltName` для каждого пользователя ClickHouse в файле настройки `config.xml`:
**Example**
```xml
-
-
+
+
host.domain.com:example_user
host.domain.com:example_user_dev
-
+
-
+
+
+
+ DNS:host.domain.com
+
+
+
+
```
diff --git a/docs/ru/sql-reference/functions/rounding-functions.md b/docs/ru/sql-reference/functions/rounding-functions.md
index cc939f69afc..f1dd57505ea 100644
--- a/docs/ru/sql-reference/functions/rounding-functions.md
+++ b/docs/ru/sql-reference/functions/rounding-functions.md
@@ -10,7 +10,7 @@ sidebar_label: "Функции округления"
Возвращает наибольшее круглое число, которое меньше или равно, чем x.
Круглым называется число, кратное 1 / 10N или ближайшее к нему число соответствующего типа данных, если 1 / 10N не представимо точно.
-N - целочисленная константа, не обязательный параметр. По умолчанию - ноль, что означает - округлять до целого числа.
+N - целочисленный аргумент, не обязательный параметр. По умолчанию - ноль, что означает - округлять до целого числа.
N может быть отрицательным.
Примеры: `floor(123.45, 1) = 123.4, floor(123.45, -1) = 120.`
diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md
index fc258f7b4cf..2436581fc7f 100644
--- a/docs/ru/sql-reference/functions/string-functions.md
+++ b/docs/ru/sql-reference/functions/string-functions.md
@@ -538,16 +538,28 @@ SELECT base58Decode('3dc8KtHrwM');
Синоним: `TO_BASE64`.
+## base64URLEncode(s)
+
+Производит кодирование URL (String или FixedString) в base64-представление в соответствии с [RFC 4648](https://tools.ietf.org/html/rfc4648).
+
## base64Decode(s) {#base64decode}
Декодирует base64-представление s в исходную строку. При невозможности декодирования выбрасывает исключение
Синоним: `FROM_BASE64`.
+## base64URLDecode(s)
+
+Декодирует base64-представление URL в исходную строку в соответствии с [RFC 4648](https://tools.ietf.org/html/rfc4648). При невозможности декодирования выбрасывает исключение
+
## tryBase64Decode(s) {#trybase64decode}
Функционал аналогичен base64Decode, но при невозможности декодирования возвращает пустую строку.
+## tryBase64URLDecode(s)
+
+Функционал аналогичен base64URLDecode, но при невозможности декодирования возвращает пустую строку.
+
## endsWith(s, suffix) {#endswith}
Возвращает 1, если строка завершается указанным суффиксом, и 0 в противном случае.
diff --git a/docs/ru/sql-reference/statements/alter/constraint.md b/docs/ru/sql-reference/statements/alter/constraint.md
index ad5f23e5fdc..45b0f5f6350 100644
--- a/docs/ru/sql-reference/statements/alter/constraint.md
+++ b/docs/ru/sql-reference/statements/alter/constraint.md
@@ -11,8 +11,8 @@ sidebar_label: "Манипуляции с ограничениями"
Добавить или удалить ограничение можно с помощью запросов
``` sql
-ALTER TABLE [db].name [ON CLUSTER cluster] ADD CONSTRAINT constraint_name CHECK expression;
-ALTER TABLE [db].name [ON CLUSTER cluster] DROP CONSTRAINT constraint_name;
+ALTER TABLE [db].name [ON CLUSTER cluster] ADD CONSTRAINT [IF NOT EXISTS] constraint_name CHECK expression;
+ALTER TABLE [db].name [ON CLUSTER cluster] DROP CONSTRAINT [IF EXISTS] constraint_name;
```
Запросы выполняют добавление или удаление метаданных об ограничениях таблицы `[db].name`, поэтому выполняются мгновенно.
diff --git a/docs/ru/sql-reference/statements/create/user.md b/docs/ru/sql-reference/statements/create/user.md
index 76cfdb251dc..fac2cacf8cc 100644
--- a/docs/ru/sql-reference/statements/create/user.md
+++ b/docs/ru/sql-reference/statements/create/user.md
@@ -13,7 +13,7 @@ sidebar_label: "Пользователь"
``` sql
CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
[, name2 [ON CLUSTER cluster_name2] ...]
- [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'}]
+ [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'}]
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
[DEFAULT ROLE role [,...]]
[DEFAULT DATABASE database | NONE]
diff --git a/docs/zh/engines/table-engines/mergetree-family/mergetree.md b/docs/zh/engines/table-engines/mergetree-family/mergetree.md
index 67bd681269b..d5ece5b23a9 100644
--- a/docs/zh/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/zh/engines/table-engines/mergetree-family/mergetree.md
@@ -201,18 +201,18 @@ ClickHouse 不要求主键唯一，所以您可以插入多条具有相同主键
主键中列的数量并没有明确的限制。依据数据结构，您可以在主键包含多些或少些列。这样可以：
- - 改善索引的性能。
+- 改善索引的性能。
- - 如果当前主键是 `(a, b)` ，在下列情况下添加另一个 `c` 列会提升性能：
+ 如果当前主键是 `(a, b)` ，在下列情况下添加另一个 `c` 列会提升性能：
- - 查询会使用 `c` 列作为条件
- - 很长的数据范围（ `index_granularity` 的数倍）里 `(a, b)` 都是相同的值，并且这样的情况很普遍。换言之，就是加入另一列后，可以让您的查询略过很长的数据范围。
+ - 查询会使用 `c` 列作为条件
+ - 很长的数据范围（ `index_granularity` 的数倍）里 `(a, b)` 都是相同的值，并且这样的情况很普遍。换言之，就是加入另一列后，可以让您的查询略过很长的数据范围。
- - 改善数据压缩。
+- 改善数据压缩。
- ClickHouse 以主键排序片段数据，所以，数据的一致性越高，压缩越好。
+ ClickHouse 以主键排序片段数据，所以，数据的一致性越高，压缩越好。
- - 在[CollapsingMergeTree](collapsingmergetree.md#table_engine-collapsingmergetree) 和 [SummingMergeTree](summingmergetree.md) 引擎里进行数据合并时会提供额外的处理逻辑。
+- 在[CollapsingMergeTree](collapsingmergetree.md#table_engine-collapsingmergetree) 和 [SummingMergeTree](summingmergetree.md) 引擎里进行数据合并时会提供额外的处理逻辑。
在这种情况下，指定与主键不同的 *排序键* 也是有意义的。
diff --git a/docs/zh/guides/improving-query-performance/skipping-indexes.md b/docs/zh/guides/improving-query-performance/skipping-indexes.md
index f9f43e46927..8eb88d859f2 100644
--- a/docs/zh/guides/improving-query-performance/skipping-indexes.md
+++ b/docs/zh/guides/improving-query-performance/skipping-indexes.md
@@ -123,7 +123,7 @@ Bloom filter是一种数据结构，它允许对集合成员进行高效的是
有三种基于Bloom过滤器的数据跳数索引类型：
-* 基本的**bloom_filter**接受一个可选参数，该参数表示在0到1之间允许的“假阳性”率(如果未指定，则使用.025)。
+* 基本的**bloom_filter**接受一个可选参数，该参数表示在0到1之间允许的“假阳性”率(如果未指定，则使用0.025)。
* 更专业的**tokenbf_v1**。需要三个参数，用来优化布隆过滤器：（1）过滤器的大小字节(大过滤器有更少的假阳性，有更高的存储成本)，（2）哈希函数的个数(更多的散列函数可以减少假阳性)。（3）布隆过滤器哈希函数的种子。有关这些参数如何影响布隆过滤器功能的更多细节，请参阅 [这里](https://hur.st/bloomfilter/) 。此索引仅适用于String、FixedString和Map类型的数据。输入表达式被分割为由非字母数字字符分隔的字符序列。例如，列值`This is a candidate for a "full text" search`将被分割为`This` `is` `a` `candidate` `for` `full` `text` `search`。它用于LIKE、EQUALS、in、hasToken()和类似的长字符串中单词和其他值的搜索。例如，一种可能的用途是在非结构的应用程序日志行列中搜索少量的类名或行号。
diff --git a/docs/zh/sql-reference/statements/alter/constraint.md b/docs/zh/sql-reference/statements/alter/constraint.md
index 86ffcf09d65..59edcf10645 100644
--- a/docs/zh/sql-reference/statements/alter/constraint.md
+++ b/docs/zh/sql-reference/statements/alter/constraint.md
@@ -9,8 +9,8 @@ sidebar_label: 约束
约束可以使用以下语法添加或删除:
``` sql
-ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression;
-ALTER TABLE [db].name DROP CONSTRAINT constraint_name;
+ALTER TABLE [db].name [ON CLUSTER cluster] ADD CONSTRAINT [IF NOT EXISTS] constraint_name CHECK expression;
+ALTER TABLE [db].name [ON CLUSTER cluster] DROP CONSTRAINT [IF EXISTS] constraint_name;
```
查看[constraints](../../../sql-reference/statements/create/table.mdx#constraints)。
diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index efe23d57478..6343dc85d00 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -64,6 +64,7 @@ namespace ErrorCodes
extern const int NETWORK_ERROR;
extern const int AUTHENTICATION_FAILED;
extern const int NO_ELEMENTS_IN_CONFIG;
+ extern const int USER_EXPIRED;
}
@@ -74,6 +75,12 @@ void Client::processError(const String & query) const
fmt::print(stderr, "Received exception from server (version {}):\n{}\n",
server_version,
getExceptionMessage(*server_exception, print_stack_trace, true));
+
+ if (server_exception->code() == ErrorCodes::USER_EXPIRED)
+ {
+ server_exception->rethrow();
+ }
+
if (is_interactive)
{
fmt::print(stderr, "\n");
@@ -241,6 +248,10 @@ std::vector Client::loadWarningMessages()
}
}
+Poco::Util::LayeredConfiguration & Client::getClientConfiguration()
+{
+ return config();
+}
void Client::initialize(Poco::Util::Application & self)
{
@@ -690,9 +701,7 @@ bool Client::processWithFuzzing(const String & full_query)
const char * begin = full_query.data();
orig_ast = parseQuery(begin, begin + full_query.size(),
global_context->getSettingsRef(),
- /*allow_multi_statements=*/ true,
- /*is_interactive=*/ is_interactive,
- /*ignore_error=*/ ignore_error);
+ /*allow_multi_statements=*/ true);
}
catch (const Exception & e)
{
@@ -944,6 +953,7 @@ void Client::addOptions(OptionsDescription & options_description)
("ssh-key-file", po::value(), "File containing the SSH private key for authenticate with the server.")
("ssh-key-passphrase", po::value(), "Passphrase for the SSH private key specified by --ssh-key-file.")
("quota_key", po::value(), "A string to differentiate quotas when the user have keyed quotas configured on server")
+ ("jwt", po::value(), "Use JWT for authentication")
("max_client_network_bandwidth", po::value(), "the maximum speed of data exchange over the network for the client in bytes per second.")
("compression", po::value(), "enable or disable compression (enabled by default for remote communication and disabled for localhost communication).")
@@ -1102,6 +1112,12 @@ void Client::processOptions(const OptionsDescription & options_description,
config().setBool("no-warnings", true);
if (options.count("fake-drop"))
config().setString("ignore_drop_queries_probability", "1");
+ if (options.count("jwt"))
+ {
+ if (!options["user"].defaulted())
+ throw Exception(ErrorCodes::BAD_ARGUMENTS, "User and JWT flags can't be specified together");
+ config().setString("jwt", options["jwt"].as());
+ }
if (options.count("accept-invalid-certificate"))
{
config().setString("openSSL.client.invalidCertificateHandler.name", "AcceptCertificateHandler");
diff --git a/programs/client/Client.h b/programs/client/Client.h
index bef948b3c1e..229608f787d 100644
--- a/programs/client/Client.h
+++ b/programs/client/Client.h
@@ -16,6 +16,9 @@ public:
int main(const std::vector & /*args*/) override;
protected:
+
+ Poco::Util::LayeredConfiguration & getClientConfiguration() override;
+
bool processWithFuzzing(const String & full_query) override;
std::optional processFuzzingStep(const String & query_to_execute, const ASTPtr & parsed_query);
diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp
index ebec337060c..a20c1f686f3 100644
--- a/programs/keeper-client/KeeperClient.cpp
+++ b/programs/keeper-client/KeeperClient.cpp
@@ -368,7 +368,7 @@ int KeeperClient::main(const std::vector & /* args */)
return 0;
}
- DB::ConfigProcessor config_processor(config().getString("config-file", "config.xml"));
+ ConfigProcessor config_processor(config().getString("config-file", "config.xml"));
/// This will handle a situation when clickhouse is running on the embedded config, but config.d folder is also present.
ConfigProcessor::registerEmbeddedConfig("config.xml", "");
@@ -383,6 +383,9 @@ int KeeperClient::main(const std::vector & /* args */)
for (const auto & key : keys)
{
+ if (key != "node")
+ continue;
+
String prefix = "zookeeper." + key;
String host = clickhouse_config.configuration->getString(prefix + ".host");
String port = clickhouse_config.configuration->getString(prefix + ".port");
@@ -401,6 +404,7 @@ int KeeperClient::main(const std::vector & /* args */)
zk_args.hosts.push_back(host + ":" + port);
}
+ zk_args.availability_zones.resize(zk_args.hosts.size());
zk_args.connection_timeout_ms = config().getInt("connection-timeout", 10) * 1000;
zk_args.session_timeout_ms = config().getInt("session-timeout", 10) * 1000;
zk_args.operation_timeout_ms = config().getInt("operation-timeout", 10) * 1000;
diff --git a/programs/keeper-client/Parser.cpp b/programs/keeper-client/Parser.cpp
index 5b16e6d2c23..51f85cf4a69 100644
--- a/programs/keeper-client/Parser.cpp
+++ b/programs/keeper-client/Parser.cpp
@@ -12,8 +12,7 @@ bool parseKeeperArg(IParser::Pos & pos, Expected & expected, String & result)
if (!parseIdentifierOrStringLiteral(pos, expected, result))
return false;
}
-
- while (pos->type != TokenType::Whitespace && pos->type != TokenType::EndOfStream && pos->type != TokenType::Semicolon)
+ else if (pos->type == TokenType::Number)
{
result.append(pos->begin, pos->end);
++pos;
@@ -40,8 +39,8 @@ bool KeeperParser::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
for (const auto & pair : KeeperClient::commands)
expected.add(pos, pair.first.data());
- for (const auto & flwc : four_letter_word_commands)
- expected.add(pos, flwc.data());
+ for (const auto & four_letter_word_command : four_letter_word_commands)
+ expected.add(pos, four_letter_word_command.data());
if (pos->type != TokenType::BareWord)
return false;
diff --git a/programs/keeper-client/Parser.h b/programs/keeper-client/Parser.h
index 57ee6ce4a18..503edfa4f73 100644
--- a/programs/keeper-client/Parser.h
+++ b/programs/keeper-client/Parser.h
@@ -11,7 +11,6 @@ namespace DB
{
bool parseKeeperArg(IParser::Pos & pos, Expected & expected, String & result);
-
bool parseKeeperPath(IParser::Pos & pos, Expected & expected, String & path);
diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt
index 22874e199be..91e162e40af 100644
--- a/programs/keeper/CMakeLists.txt
+++ b/programs/keeper/CMakeLists.txt
@@ -154,8 +154,6 @@ if (BUILD_STANDALONE_KEEPER)
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/ThreadPoolRemoteFSReader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/ThreadPoolReader.cpp
- ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Storages/StorageS3Settings.cpp
-
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/BaseDaemon.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/SentryWriter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/GraphiteWriter.cpp
diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp
index dba5c2b7d2a..f14ef2e5552 100644
--- a/programs/keeper/Keeper.cpp
+++ b/programs/keeper/Keeper.cpp
@@ -355,15 +355,13 @@ try
std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
- if (config().has(DB::PlacementInfo::PLACEMENT_CONFIG_PREFIX))
- {
- PlacementInfo::PlacementInfo::instance().initialize(config());
- }
+ PlacementInfo::PlacementInfo::instance().initialize(config());
GlobalThreadPool::initialize(
- config().getUInt("max_thread_pool_size", 100),
- config().getUInt("max_thread_pool_free_size", 1000),
- config().getUInt("thread_pool_queue_size", 10000)
+ /// We need to have sufficient amount of threads for connections + nuraft workers + keeper workers, 1000 is an estimation
+ std::min(1000U, config().getUInt("max_thread_pool_size", 1000)),
+ config().getUInt("max_thread_pool_free_size", 100),
+ config().getUInt("thread_pool_queue_size", 1000)
);
/// Wait for all threads to avoid possible use-after-free (for example logging objects can be already destroyed).
SCOPE_EXIT({
@@ -576,8 +574,7 @@ try
#if USE_SSL
CertificateReloader::instance().tryLoad(*config);
#endif
- },
- /* already_loaded = */ false); /// Reload it right now (initial loading)
+ });
SCOPE_EXIT({
LOG_INFO(log, "Shutting down.");
diff --git a/programs/library-bridge/CMakeLists.txt b/programs/library-bridge/CMakeLists.txt
index 2fca10ce4d7..86410d712ec 100644
--- a/programs/library-bridge/CMakeLists.txt
+++ b/programs/library-bridge/CMakeLists.txt
@@ -11,7 +11,6 @@ set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES
LibraryBridgeHandlers.cpp
SharedLibrary.cpp
library-bridge.cpp
- createFunctionBaseCast.cpp
)
clickhouse_add_executable(clickhouse-library-bridge ${CLICKHOUSE_LIBRARY_BRIDGE_SOURCES})
@@ -20,6 +19,7 @@ target_link_libraries(clickhouse-library-bridge PRIVATE
daemon
dbms
bridge
+ clickhouse_functions
)
set_target_properties(clickhouse-library-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 4d5cfb09e6a..b33e1595056 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -32,6 +32,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -59,8 +60,13 @@
# include
#endif
+
namespace fs = std::filesystem;
+namespace CurrentMetrics
+{
+ extern const Metric MemoryTracking;
+}
namespace DB
{
@@ -82,6 +88,11 @@ void applySettingsOverridesForLocal(ContextMutablePtr context)
context->setSettings(settings);
}
+Poco::Util::LayeredConfiguration & LocalServer::getClientConfiguration()
+{
+ return config();
+}
+
void LocalServer::processError(const String &) const
{
if (ignore_error)
@@ -117,20 +128,21 @@ void LocalServer::initialize(Poco::Util::Application & self)
Poco::Util::Application::initialize(self);
/// Load config files if exists
- if (config().has("config-file") || fs::exists("config.xml"))
+ if (getClientConfiguration().has("config-file") || fs::exists("config.xml"))
{
- const auto config_path = config().getString("config-file", "config.xml");
+ const auto config_path = getClientConfiguration().getString("config-file", "config.xml");
ConfigProcessor config_processor(config_path, false, true);
ConfigProcessor::setConfigPath(fs::path(config_path).parent_path());
auto loaded_config = config_processor.loadConfig();
- config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
+ getClientConfiguration().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
}
+ server_settings.loadSettingsFromConfig(config());
+
GlobalThreadPool::initialize(
- config().getUInt("max_thread_pool_size", 10000),
- config().getUInt("max_thread_pool_free_size", 1000),
- config().getUInt("thread_pool_queue_size", 10000)
- );
+ server_settings.max_thread_pool_size,
+ server_settings.max_thread_pool_free_size,
+ server_settings.thread_pool_queue_size);
#if USE_AZURE_BLOB_STORAGE
/// See the explanation near the same line in Server.cpp
@@ -141,18 +153,17 @@ void LocalServer::initialize(Poco::Util::Application & self)
#endif
getIOThreadPool().initialize(
- config().getUInt("max_io_thread_pool_size", 100),
- config().getUInt("max_io_thread_pool_free_size", 0),
- config().getUInt("io_thread_pool_queue_size", 10000));
+ server_settings.max_io_thread_pool_size,
+ server_settings.max_io_thread_pool_free_size,
+ server_settings.io_thread_pool_queue_size);
-
- const size_t active_parts_loading_threads = config().getUInt("max_active_parts_loading_thread_pool_size", 64);
+ const size_t active_parts_loading_threads = server_settings.max_active_parts_loading_thread_pool_size;
getActivePartsLoadingThreadPool().initialize(
active_parts_loading_threads,
0, // We don't need any threads one all the parts will be loaded
active_parts_loading_threads);
- const size_t outdated_parts_loading_threads = config().getUInt("max_outdated_parts_loading_thread_pool_size", 32);
+ const size_t outdated_parts_loading_threads = server_settings.max_outdated_parts_loading_thread_pool_size;
getOutdatedPartsLoadingThreadPool().initialize(
outdated_parts_loading_threads,
0, // We don't need any threads one all the parts will be loaded
@@ -160,7 +171,7 @@ void LocalServer::initialize(Poco::Util::Application & self)
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads);
- const size_t unexpected_parts_loading_threads = config().getUInt("max_unexpected_parts_loading_thread_pool_size", 32);
+ const size_t unexpected_parts_loading_threads = server_settings.max_unexpected_parts_loading_thread_pool_size;
getUnexpectedPartsLoadingThreadPool().initialize(
unexpected_parts_loading_threads,
0, // We don't need any threads one all the parts will be loaded
@@ -168,7 +179,7 @@ void LocalServer::initialize(Poco::Util::Application & self)
getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads);
- const size_t cleanup_threads = config().getUInt("max_parts_cleaning_thread_pool_size", 128);
+ const size_t cleanup_threads = server_settings.max_parts_cleaning_thread_pool_size;
getPartsCleaningThreadPool().initialize(
cleanup_threads,
0, // We don't need any threads one all the parts will be deleted
@@ -201,10 +212,10 @@ void LocalServer::tryInitPath()
{
std::string path;
- if (config().has("path"))
+ if (getClientConfiguration().has("path"))
{
// User-supplied path.
- path = config().getString("path");
+ path = getClientConfiguration().getString("path");
Poco::trimInPlace(path);
if (path.empty())
@@ -263,13 +274,13 @@ void LocalServer::tryInitPath()
global_context->setUserFilesPath(""); /// user's files are everywhere
- std::string user_scripts_path = config().getString("user_scripts_path", fs::path(path) / "user_scripts/");
+ std::string user_scripts_path = getClientConfiguration().getString("user_scripts_path", fs::path(path) / "user_scripts/");
global_context->setUserScriptsPath(user_scripts_path);
/// top_level_domains_lists
- const std::string & top_level_domains_path = config().getString("top_level_domains_path", fs::path(path) / "top_level_domains/");
+ const std::string & top_level_domains_path = getClientConfiguration().getString("top_level_domains_path", fs::path(path) / "top_level_domains/");
if (!top_level_domains_path.empty())
- TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config());
+ TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", getClientConfiguration());
}
@@ -311,14 +322,14 @@ void LocalServer::cleanup()
std::string LocalServer::getInitialCreateTableQuery()
{
- if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!isRegularFile(STDIN_FILENO) || queries.empty()))
+ if (!getClientConfiguration().has("table-structure") && !getClientConfiguration().has("table-file") && !getClientConfiguration().has("table-data-format") && (!isRegularFile(STDIN_FILENO) || queries.empty()))
return {};
- auto table_name = backQuoteIfNeed(config().getString("table-name", "table"));
- auto table_structure = config().getString("table-structure", "auto");
+ auto table_name = backQuoteIfNeed(getClientConfiguration().getString("table-name", "table"));
+ auto table_structure = getClientConfiguration().getString("table-structure", "auto");
String table_file;
- if (!config().has("table-file") || config().getString("table-file") == "-")
+ if (!getClientConfiguration().has("table-file") || getClientConfiguration().getString("table-file") == "-")
{
/// Use Unix tools stdin naming convention
table_file = "stdin";
@@ -326,7 +337,7 @@ std::string LocalServer::getInitialCreateTableQuery()
else
{
/// Use regular file
- auto file_name = config().getString("table-file");
+ auto file_name = getClientConfiguration().getString("table-file");
table_file = quoteString(file_name);
}
@@ -374,18 +385,18 @@ void LocalServer::setupUsers()
ConfigurationPtr users_config;
auto & access_control = global_context->getAccessControl();
- access_control.setNoPasswordAllowed(config().getBool("allow_no_password", true));
- access_control.setPlaintextPasswordAllowed(config().getBool("allow_plaintext_password", true));
- if (config().has("config-file") || fs::exists("config.xml"))
+ access_control.setNoPasswordAllowed(getClientConfiguration().getBool("allow_no_password", true));
+ access_control.setPlaintextPasswordAllowed(getClientConfiguration().getBool("allow_plaintext_password", true));
+ if (getClientConfiguration().has("config-file") || fs::exists("config.xml"))
{
- String config_path = config().getString("config-file", "");
- bool has_user_directories = config().has("user_directories");
+ String config_path = getClientConfiguration().getString("config-file", "");
+ bool has_user_directories = getClientConfiguration().has("user_directories");
const auto config_dir = fs::path{config_path}.remove_filename().string();
- String users_config_path = config().getString("users_config", "");
+ String users_config_path = getClientConfiguration().getString("users_config", "");
if (users_config_path.empty() && has_user_directories)
{
- users_config_path = config().getString("user_directories.users_xml.path");
+ users_config_path = getClientConfiguration().getString("user_directories.users_xml.path");
if (fs::path(users_config_path).is_relative() && fs::exists(fs::path(config_dir) / users_config_path))
users_config_path = fs::path(config_dir) / users_config_path;
}
@@ -409,10 +420,10 @@ void LocalServer::setupUsers()
void LocalServer::connect()
{
- connection_parameters = ConnectionParameters(config(), "localhost");
+ connection_parameters = ConnectionParameters(getClientConfiguration(), "localhost");
ReadBuffer * in;
- auto table_file = config().getString("table-file", "-");
+ auto table_file = getClientConfiguration().getString("table-file", "-");
if (table_file == "-" || table_file == "stdin")
{
in = &std_in;
@@ -433,7 +444,7 @@ try
UseSSL use_ssl;
thread_status.emplace();
- StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true));
+ StackTrace::setShowAddresses(server_settings.show_addresses_in_stack_traces);
setupSignalHandler();
@@ -448,7 +459,7 @@ try
if (rlim.rlim_cur < rlim.rlim_max)
{
- rlim.rlim_cur = config().getUInt("max_open_files", static_cast(rlim.rlim_max));
+ rlim.rlim_cur = getClientConfiguration().getUInt("max_open_files", static_cast(rlim.rlim_max));
int rc = setrlimit(RLIMIT_NOFILE, &rlim);
if (rc != 0)
std::cerr << fmt::format("Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, errnoToString()) << '\n';
@@ -456,8 +467,8 @@ try
}
is_interactive = stdin_is_a_tty
- && (config().hasOption("interactive")
- || (queries.empty() && !config().has("table-structure") && queries_files.empty() && !config().has("table-file")));
+ && (getClientConfiguration().hasOption("interactive")
+ || (queries.empty() && !getClientConfiguration().has("table-structure") && queries_files.empty() && !getClientConfiguration().has("table-file")));
if (!is_interactive)
{
@@ -481,7 +492,7 @@ try
SCOPE_EXIT({ cleanup(); });
- initTTYBuffer(toProgressOption(config().getString("progress", "default")));
+ initTTYBuffer(toProgressOption(getClientConfiguration().getString("progress", "default")));
ASTAlterCommand::setFormatAlterCommandsWithParentheses(true);
applyCmdSettings(global_context);
@@ -489,7 +500,7 @@ try
/// try to load user defined executable functions, throw on error and die
try
{
- global_context->loadOrReloadUserDefinedExecutableFunctions(config());
+ global_context->loadOrReloadUserDefinedExecutableFunctions(getClientConfiguration());
}
catch (...)
{
@@ -530,7 +541,7 @@ try
}
catch (const DB::Exception & e)
{
- bool need_print_stack_trace = config().getBool("stacktrace", false);
+ bool need_print_stack_trace = getClientConfiguration().getBool("stacktrace", false);
std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl;
return e.code() ? e.code() : -1;
}
@@ -542,42 +553,42 @@ catch (...)
void LocalServer::updateLoggerLevel(const String & logs_level)
{
- config().setString("logger.level", logs_level);
- updateLevels(config(), logger());
+ getClientConfiguration().setString("logger.level", logs_level);
+ updateLevels(getClientConfiguration(), logger());
}
void LocalServer::processConfig()
{
- if (!queries.empty() && config().has("queries-file"))
+ if (!queries.empty() && getClientConfiguration().has("queries-file"))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Options '--query' and '--queries-file' cannot be specified at the same time");
- if (config().has("multiquery"))
+ if (getClientConfiguration().has("multiquery"))
is_multiquery = true;
- pager = config().getString("pager", "");
+ pager = getClientConfiguration().getString("pager", "");
- delayed_interactive = config().has("interactive") && (!queries.empty() || config().has("queries-file"));
+ delayed_interactive = getClientConfiguration().has("interactive") && (!queries.empty() || getClientConfiguration().has("queries-file"));
if (!is_interactive || delayed_interactive)
{
- echo_queries = config().hasOption("echo") || config().hasOption("verbose");
- ignore_error = config().getBool("ignore-error", false);
+ echo_queries = getClientConfiguration().hasOption("echo") || getClientConfiguration().hasOption("verbose");
+ ignore_error = getClientConfiguration().getBool("ignore-error", false);
}
- print_stack_trace = config().getBool("stacktrace", false);
+ print_stack_trace = getClientConfiguration().getBool("stacktrace", false);
const std::string clickhouse_dialect{"clickhouse"};
- load_suggestions = (is_interactive || delayed_interactive) && !config().getBool("disable_suggestion", false)
- && config().getString("dialect", clickhouse_dialect) == clickhouse_dialect;
- wait_for_suggestions_to_load = config().getBool("wait_for_suggestions_to_load", false);
+ load_suggestions = (is_interactive || delayed_interactive) && !getClientConfiguration().getBool("disable_suggestion", false)
+ && getClientConfiguration().getString("dialect", clickhouse_dialect) == clickhouse_dialect;
+ wait_for_suggestions_to_load = getClientConfiguration().getBool("wait_for_suggestions_to_load", false);
- auto logging = (config().has("logger.console")
- || config().has("logger.level")
- || config().has("log-level")
- || config().has("send_logs_level")
- || config().has("logger.log"));
+ auto logging = (getClientConfiguration().has("logger.console")
+ || getClientConfiguration().has("logger.level")
+ || getClientConfiguration().has("log-level")
+ || getClientConfiguration().has("send_logs_level")
+ || getClientConfiguration().has("logger.log"));
- auto level = config().getString("log-level", "trace");
+ auto level = getClientConfiguration().getString("log-level", "trace");
- if (config().has("server_logs_file"))
+ if (getClientConfiguration().has("server_logs_file"))
{
auto poco_logs_level = Poco::Logger::parseLevel(level);
Poco::Logger::root().setLevel(poco_logs_level);
@@ -587,10 +598,10 @@ void LocalServer::processConfig()
}
else
{
- config().setString("logger", "logger");
+ getClientConfiguration().setString("logger", "logger");
auto log_level_default = logging ? level : "fatal";
- config().setString("logger.level", config().getString("log-level", config().getString("send_logs_level", log_level_default)));
- buildLoggers(config(), logger(), "clickhouse-local");
+ getClientConfiguration().setString("logger.level", getClientConfiguration().getString("log-level", getClientConfiguration().getString("send_logs_level", log_level_default)));
+ buildLoggers(getClientConfiguration(), logger(), "clickhouse-local");
}
shared_context = Context::createShared();
@@ -604,13 +615,13 @@ void LocalServer::processConfig()
LoggerRawPtr log = &logger();
/// Maybe useless
- if (config().has("macros"))
- global_context->setMacros(std::make_unique(config(), "macros", log));
+ if (getClientConfiguration().has("macros"))
+ global_context->setMacros(std::make_unique(getClientConfiguration(), "macros", log));
setDefaultFormatsAndCompressionFromConfiguration();
/// Sets external authenticators config (LDAP, Kerberos).
- global_context->setExternalAuthenticatorsConfig(config());
+ global_context->setExternalAuthenticatorsConfig(getClientConfiguration());
setupUsers();
@@ -619,12 +630,43 @@ void LocalServer::processConfig()
global_context->getProcessList().setMaxSize(0);
const size_t physical_server_memory = getMemoryAmount();
- const double cache_size_to_ram_max_ratio = config().getDouble("cache_size_to_ram_max_ratio", 0.5);
+
+ size_t max_server_memory_usage = server_settings.max_server_memory_usage;
+ double max_server_memory_usage_to_ram_ratio = server_settings.max_server_memory_usage_to_ram_ratio;
+
+ size_t default_max_server_memory_usage = static_cast(physical_server_memory * max_server_memory_usage_to_ram_ratio);
+
+ if (max_server_memory_usage == 0)
+ {
+ max_server_memory_usage = default_max_server_memory_usage;
+ LOG_INFO(log, "Setting max_server_memory_usage was set to {}"
+ " ({} available * {:.2f} max_server_memory_usage_to_ram_ratio)",
+ formatReadableSizeWithBinarySuffix(max_server_memory_usage),
+ formatReadableSizeWithBinarySuffix(physical_server_memory),
+ max_server_memory_usage_to_ram_ratio);
+ }
+ else if (max_server_memory_usage > default_max_server_memory_usage)
+ {
+ max_server_memory_usage = default_max_server_memory_usage;
+ LOG_INFO(log, "Setting max_server_memory_usage was lowered to {}"
+ " because the system has low amount of memory. The amount was"
+ " calculated as {} available"
+ " * {:.2f} max_server_memory_usage_to_ram_ratio",
+ formatReadableSizeWithBinarySuffix(max_server_memory_usage),
+ formatReadableSizeWithBinarySuffix(physical_server_memory),
+ max_server_memory_usage_to_ram_ratio);
+ }
+
+ total_memory_tracker.setHardLimit(max_server_memory_usage);
+ total_memory_tracker.setDescription("(total)");
+ total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
+
+ const double cache_size_to_ram_max_ratio = server_settings.cache_size_to_ram_max_ratio;
const size_t max_cache_size = static_cast(physical_server_memory * cache_size_to_ram_max_ratio);
- String uncompressed_cache_policy = config().getString("uncompressed_cache_policy", DEFAULT_UNCOMPRESSED_CACHE_POLICY);
- size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", DEFAULT_UNCOMPRESSED_CACHE_MAX_SIZE);
- double uncompressed_cache_size_ratio = config().getDouble("uncompressed_cache_size_ratio", DEFAULT_UNCOMPRESSED_CACHE_SIZE_RATIO);
+ String uncompressed_cache_policy = server_settings.uncompressed_cache_policy;
+ size_t uncompressed_cache_size = server_settings.uncompressed_cache_size;
+ double uncompressed_cache_size_ratio = server_settings.uncompressed_cache_size_ratio;
if (uncompressed_cache_size > max_cache_size)
{
uncompressed_cache_size = max_cache_size;
@@ -632,9 +674,9 @@ void LocalServer::processConfig()
}
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio);
- String mark_cache_policy = config().getString("mark_cache_policy", DEFAULT_MARK_CACHE_POLICY);
- size_t mark_cache_size = config().getUInt64("mark_cache_size", DEFAULT_MARK_CACHE_MAX_SIZE);
- double mark_cache_size_ratio = config().getDouble("mark_cache_size_ratio", DEFAULT_MARK_CACHE_SIZE_RATIO);
+ String mark_cache_policy = server_settings.mark_cache_policy;
+ size_t mark_cache_size = server_settings.mark_cache_size;
+ double mark_cache_size_ratio = server_settings.mark_cache_size_ratio;
if (!mark_cache_size)
LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation.");
if (mark_cache_size > max_cache_size)
@@ -644,9 +686,9 @@ void LocalServer::processConfig()
}
global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio);
- String index_uncompressed_cache_policy = config().getString("index_uncompressed_cache_policy", DEFAULT_INDEX_UNCOMPRESSED_CACHE_POLICY);
- size_t index_uncompressed_cache_size = config().getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE);
- double index_uncompressed_cache_size_ratio = config().getDouble("index_uncompressed_cache_size_ratio", DEFAULT_INDEX_UNCOMPRESSED_CACHE_SIZE_RATIO);
+ String index_uncompressed_cache_policy = server_settings.index_uncompressed_cache_policy;
+ size_t index_uncompressed_cache_size = server_settings.index_uncompressed_cache_size;
+ double index_uncompressed_cache_size_ratio = server_settings.index_uncompressed_cache_size_ratio;
if (index_uncompressed_cache_size > max_cache_size)
{
index_uncompressed_cache_size = max_cache_size;
@@ -654,9 +696,9 @@ void LocalServer::processConfig()
}
global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio);
- String index_mark_cache_policy = config().getString("index_mark_cache_policy", DEFAULT_INDEX_MARK_CACHE_POLICY);
- size_t index_mark_cache_size = config().getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE);
- double index_mark_cache_size_ratio = config().getDouble("index_mark_cache_size_ratio", DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO);
+ String index_mark_cache_policy = server_settings.index_mark_cache_policy;
+ size_t index_mark_cache_size = server_settings.index_mark_cache_size;
+ double index_mark_cache_size_ratio = server_settings.index_mark_cache_size_ratio;
if (index_mark_cache_size > max_cache_size)
{
index_mark_cache_size = max_cache_size;
@@ -664,7 +706,7 @@ void LocalServer::processConfig()
}
global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio);
- size_t mmap_cache_size = config().getUInt64("mmap_cache_size", DEFAULT_MMAP_CACHE_MAX_SIZE);
+ size_t mmap_cache_size = server_settings.mmap_cache_size;
if (mmap_cache_size > max_cache_size)
{
mmap_cache_size = max_cache_size;
@@ -676,8 +718,8 @@ void LocalServer::processConfig()
global_context->setQueryCache(0, 0, 0, 0);
#if USE_EMBEDDED_COMPILER
- size_t compiled_expression_cache_max_size_in_bytes = config().getUInt64("compiled_expression_cache_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_SIZE);
- size_t compiled_expression_cache_max_elements = config().getUInt64("compiled_expression_cache_elements_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_ENTRIES);
+ size_t compiled_expression_cache_max_size_in_bytes = server_settings.compiled_expression_cache_size;
+ size_t compiled_expression_cache_max_elements = server_settings.compiled_expression_cache_elements_size;
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
#endif
@@ -689,16 +731,16 @@ void LocalServer::processConfig()
applyCmdOptions(global_context);
/// Load global settings from default_profile and system_profile.
- global_context->setDefaultProfiles(config());
+ global_context->setDefaultProfiles(getClientConfiguration());
/// We load temporary database first, because projections need it.
DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase();
- std::string default_database = config().getString("default_database", "default");
+ std::string default_database = server_settings.default_database;
DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context));
global_context->setCurrentDatabase(default_database);
- if (config().has("path"))
+ if (getClientConfiguration().has("path"))
{
String path = global_context->getPath();
fs::create_directories(fs::path(path));
@@ -713,7 +755,7 @@ void LocalServer::processConfig()
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
waitLoad(TablesLoaderForegroundPoolId, startup_system_tasks);
- if (!config().has("only-system-tables"))
+ if (!getClientConfiguration().has("only-system-tables"))
{
DatabaseCatalog::instance().createBackgroundTasks();
waitLoad(loadMetadata(global_context));
@@ -725,18 +767,15 @@ void LocalServer::processConfig()
LOG_DEBUG(log, "Loaded metadata.");
}
- else if (!config().has("no-system-tables"))
+ else if (!getClientConfiguration().has("no-system-tables"))
{
attachSystemTablesServer(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE), false);
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
}
- server_display_name = config().getString("display_name", getFQDNOrHostName());
- prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", "{display_name} :) ");
- std::map prompt_substitutions{{"display_name", server_display_name}};
- for (const auto & [key, value] : prompt_substitutions)
- boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value);
+ server_display_name = getClientConfiguration().getString("display_name", "");
+ prompt_by_server_display_name = getClientConfiguration().getRawString("prompt_by_server_display_name.default", ":) ");
global_context->setQueryKindInitial();
global_context->setQueryKind(query_kind);
@@ -814,7 +853,7 @@ void LocalServer::applyCmdSettings(ContextMutablePtr context)
void LocalServer::applyCmdOptions(ContextMutablePtr context)
{
- context->setDefaultFormat(config().getString("output-format", config().getString("format", is_interactive ? "PrettyCompact" : "TSV")));
+ context->setDefaultFormat(getClientConfiguration().getString("output-format", getClientConfiguration().getString("format", is_interactive ? "PrettyCompact" : "TSV")));
applyCmdSettings(context);
}
@@ -822,33 +861,33 @@ void LocalServer::applyCmdOptions(ContextMutablePtr context)
void LocalServer::processOptions(const OptionsDescription &, const CommandLineOptions & options, const std::vector &, const std::vector &)
{
if (options.count("table"))
- config().setString("table-name", options["table"].as());
+ getClientConfiguration().setString("table-name", options["table"].as());
if (options.count("file"))
- config().setString("table-file", options["file"].as());
+ getClientConfiguration().setString("table-file", options["file"].as());
if (options.count("structure"))
- config().setString("table-structure", options["structure"].as());
+ getClientConfiguration().setString("table-structure", options["structure"].as());
if (options.count("no-system-tables"))
- config().setBool("no-system-tables", true);
+ getClientConfiguration().setBool("no-system-tables", true);
if (options.count("only-system-tables"))
- config().setBool("only-system-tables", true);
+ getClientConfiguration().setBool("only-system-tables", true);
if (options.count("database"))
- config().setString("default_database", options["database"].as());
+ getClientConfiguration().setString("default_database", options["database"].as());
if (options.count("input-format"))
- config().setString("table-data-format", options["input-format"].as());
+ getClientConfiguration().setString("table-data-format", options["input-format"].as());
if (options.count("output-format"))
- config().setString("output-format", options["output-format"].as());
+ getClientConfiguration().setString("output-format", options["output-format"].as());
if (options.count("logger.console"))
- config().setBool("logger.console", options["logger.console"].as());
+ getClientConfiguration().setBool("logger.console", options["logger.console"].as());
if (options.count("logger.log"))
- config().setString("logger.log", options["logger.log"].as());
+ getClientConfiguration().setString("logger.log", options["logger.log"].as());
if (options.count("logger.level"))
- config().setString("logger.level", options["logger.level"].as());
+ getClientConfiguration().setString("logger.level", options["logger.level"].as());
if (options.count("send_logs_level"))
- config().setString("send_logs_level", options["send_logs_level"].as());
+ getClientConfiguration().setString("send_logs_level", options["send_logs_level"].as());
if (options.count("wait_for_suggestions_to_load"))
- config().setBool("wait_for_suggestions_to_load", true);
+ getClientConfiguration().setBool("wait_for_suggestions_to_load", true);
}
void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector &, std::vector &)
diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h
index 4856e68ff9b..da2466650a7 100644
--- a/programs/local/LocalServer.h
+++ b/programs/local/LocalServer.h
@@ -30,6 +30,9 @@ public:
int main(const std::vector & /*args*/) override;
protected:
+
+ Poco::Util::LayeredConfiguration & getClientConfiguration() override;
+
void connect() override;
void processError(const String & query) const override;
@@ -63,6 +66,8 @@ private:
void applyCmdOptions(ContextMutablePtr context);
void applyCmdSettings(ContextMutablePtr context);
+ ServerSettings server_settings;
+
std::optional status;
std::optional temporary_directory_to_delete;
diff --git a/programs/main.cpp b/programs/main.cpp
index c270388f17f..61e2bc18ed7 100644
--- a/programs/main.cpp
+++ b/programs/main.cpp
@@ -13,6 +13,7 @@
#include
+#include "config.h"
#include "config_tools.h"
#include
@@ -439,6 +440,14 @@ extern "C"
}
#endif
+/// Prevent messages from JeMalloc in the release build.
+/// Some of these messages are non-actionable for the users, such as:
+/// : Number of CPUs detected is not deterministic. Per-CPU arena disabled.
+#if USE_JEMALLOC && defined(NDEBUG) && !defined(SANITIZER)
+extern "C" void (*malloc_message)(void *, const char *s);
+__attribute__((constructor(0))) void init_je_malloc_message() { malloc_message = [](void *, const char *){}; }
+#endif
+
/// This allows to implement assert to forbid initialization of a class in static constructors.
/// Usage:
///
diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt
index 83839cc21ac..14af330f788 100644
--- a/programs/odbc-bridge/CMakeLists.txt
+++ b/programs/odbc-bridge/CMakeLists.txt
@@ -13,7 +13,6 @@ set (CLICKHOUSE_ODBC_BRIDGE_SOURCES
getIdentifierQuote.cpp
odbc-bridge.cpp
validateODBCConnectionString.cpp
- createFunctionBaseCast.cpp
)
clickhouse_add_executable(clickhouse-odbc-bridge ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
@@ -25,6 +24,7 @@ target_link_libraries(clickhouse-odbc-bridge PRIVATE
clickhouse_parsers
ch_contrib::nanodbc
ch_contrib::unixodbc
+ clickhouse_functions
)
set_target_properties(clickhouse-odbc-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
diff --git a/programs/odbc-bridge/ODBCSource.cpp b/programs/odbc-bridge/ODBCSource.cpp
index 940970f36ab..41a9813ce50 100644
--- a/programs/odbc-bridge/ODBCSource.cpp
+++ b/programs/odbc-bridge/ODBCSource.cpp
@@ -3,6 +3,7 @@
#include
#include
#include
+#include
#include
#include
@@ -47,9 +48,17 @@ Chunk ODBCSource::generate()
for (int idx = 0; idx < result.columns(); ++idx)
{
const auto & sample = description.sample_block.getByPosition(idx);
-
if (!result.is_null(idx))
- insertValue(*columns[idx], removeNullable(sample.type), description.types[idx].first, result, idx);
+ {
+ if (columns[idx]->isNullable())
+ {
+ ColumnNullable & column_nullable = assert_cast(*columns[idx]);
+ insertValue(column_nullable.getNestedColumn(), removeNullable(sample.type), description.types[idx].first, result, idx);
+ column_nullable.getNullMapData().emplace_back(0);
+ }
+ else
+ insertValue(*columns[idx], removeNullable(sample.type), description.types[idx].first, result, idx);
+ }
else
insertDefaultValue(*columns[idx], *sample.column);
}
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 8fcb9d87a93..4cb3b5f45c7 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -10,6 +10,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -48,6 +49,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -70,7 +72,6 @@
#include
#include
#include
-#include
#include
#include
#include
@@ -721,11 +722,6 @@ try
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision());
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
- Poco::ThreadPool server_pool(3, server_settings.max_connections);
- std::mutex servers_lock;
- std::vector servers;
- std::vector servers_to_start_before_tables;
-
/** Context contains all that query execution is dependent:
* settings, available functions, data types, aggregate functions, databases, ...
*/
@@ -773,7 +769,27 @@ try
LOG_INFO(log, "Available CPU instruction sets: {}", cpu_info);
#endif
- bool will_have_trace_collector = hasPHDRCache() && config().has("trace_log");
+ bool has_trace_collector = false;
+ /// Disable it if we collect test coverage information, because it will work extremely slow.
+#if !WITH_COVERAGE
+ /// Profilers cannot work reliably with any other libunwind or without PHDR cache.
+ has_trace_collector = hasPHDRCache() && config().has("trace_log");
+#endif
+
+ /// Describe multiple reasons when query profiler cannot work.
+
+#if WITH_COVERAGE
+ LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they work extremely slow with test coverage.");
+#endif
+
+#if defined(SANITIZER)
+ LOG_INFO(log, "Query Profiler disabled because they cannot work under sanitizers"
+ " when two different stack unwinding methods will interfere with each other.");
+#endif
+
+ if (!hasPHDRCache())
+ LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created"
+ " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe).");
// Initialize global thread pool. Do it before we fetch configs from zookeeper
// nodes (`from_zk`), because ZooKeeper interface uses the pool. We will
@@ -782,8 +798,39 @@ try
server_settings.max_thread_pool_size,
server_settings.max_thread_pool_free_size,
server_settings.thread_pool_queue_size,
- will_have_trace_collector ? server_settings.global_profiler_real_time_period_ns : 0,
- will_have_trace_collector ? server_settings.global_profiler_cpu_time_period_ns : 0);
+ has_trace_collector ? server_settings.global_profiler_real_time_period_ns : 0,
+ has_trace_collector ? server_settings.global_profiler_cpu_time_period_ns : 0);
+
+ if (has_trace_collector)
+ {
+ global_context->createTraceCollector();
+
+ /// Set up server-wide memory profiler (for total memory tracker).
+ if (server_settings.total_memory_profiler_step)
+ total_memory_tracker.setProfilerStep(server_settings.total_memory_profiler_step);
+
+ if (server_settings.total_memory_tracker_sample_probability > 0.0)
+ total_memory_tracker.setSampleProbability(server_settings.total_memory_tracker_sample_probability);
+
+ if (server_settings.total_memory_profiler_sample_min_allocation_size)
+ total_memory_tracker.setSampleMinAllocationSize(server_settings.total_memory_profiler_sample_min_allocation_size);
+
+ if (server_settings.total_memory_profiler_sample_max_allocation_size)
+ total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size);
+ }
+
+ Poco::ThreadPool server_pool(
+ /* minCapacity */3,
+ /* maxCapacity */server_settings.max_connections,
+ /* idleTime */60,
+ /* stackSize */POCO_THREAD_STACK_SIZE,
+ server_settings.global_profiler_real_time_period_ns,
+ server_settings.global_profiler_cpu_time_period_ns);
+
+ std::mutex servers_lock;
+ std::vector servers;
+ std::vector servers_to_start_before_tables;
+
/// Wait for all threads to avoid possible use-after-free (for example logging objects can be already destroyed).
SCOPE_EXIT({
Stopwatch watch;
@@ -944,6 +991,20 @@ try
}
}
+ std::string path_str = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH));
+ fs::path path = path_str;
+
+ /// Check that the process user id matches the owner of the data.
+ assertProcessUserMatchesDataOwner(path_str, [&](const std::string & message){ global_context->addWarningMessage(message); });
+
+ global_context->setPath(path_str);
+
+ StatusFile status{path / "status", StatusFile::write_full_info};
+
+ ServerUUID::load(path / "uuid", log);
+
+ PlacementInfo::PlacementInfo::instance().initialize(config());
+
zkutil::validateZooKeeperConfig(config());
bool has_zookeeper = zkutil::hasZooKeeperConfig(config());
@@ -955,7 +1016,7 @@ try
ConfigProcessor config_processor(config_path);
loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
main_config_zk_node_cache, main_config_zk_changed_event, /* fallback_to_preprocessed = */ true);
- config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
+ config_processor.savePreprocessedConfig(loaded_config, path_str);
config().removeConfiguration(old_configuration.get());
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
global_context->setConfig(loaded_config.configuration);
@@ -1089,19 +1150,6 @@ try
global_context->setRemoteHostFilter(config());
global_context->setHTTPHeaderFilter(config());
- std::string path_str = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH));
- fs::path path = path_str;
- std::string default_database = server_settings.default_database.toString();
-
- /// Check that the process user id matches the owner of the data.
- assertProcessUserMatchesDataOwner(path_str, [&](const std::string & message){ global_context->addWarningMessage(message); });
-
- global_context->setPath(path_str);
-
- StatusFile status{path / "status", StatusFile::write_full_info};
-
- ServerUUID::load(path / "uuid", log);
-
/// Try to increase limit on number of open files.
{
rlimit rlim;
@@ -1334,12 +1382,12 @@ try
global_context->setQueryCache(query_cache_max_size_in_bytes, query_cache_max_entries, query_cache_query_cache_max_entry_size_in_bytes, query_cache_max_entry_size_in_rows);
#if USE_EMBEDDED_COMPILER
- size_t compiled_expression_cache_max_size_in_bytes = config().getUInt64("compiled_expression_cache_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_SIZE);
- size_t compiled_expression_cache_max_elements = config().getUInt64("compiled_expression_cache_elements_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_ENTRIES);
+ size_t compiled_expression_cache_max_size_in_bytes = server_settings.compiled_expression_cache_size;
+ size_t compiled_expression_cache_max_elements = server_settings.compiled_expression_cache_elements_size;
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
#endif
- NamedCollectionUtils::loadIfNot();
+ NamedCollectionFactory::instance().loadIfNot();
/// Initialize main config reloader.
std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
@@ -1361,8 +1409,8 @@ try
tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization");
}
- const std::string cert_path = config().getString("openSSL.server.certificateFile", "");
- const std::string key_path = config().getString("openSSL.server.privateKeyFile", "");
+ std::string cert_path = config().getString("openSSL.server.certificateFile", "");
+ std::string key_path = config().getString("openSSL.server.privateKeyFile", "");
std::vector extra_paths = {include_from_path};
if (!cert_path.empty())
@@ -1370,6 +1418,18 @@ try
if (!key_path.empty())
extra_paths.emplace_back(key_path);
+ Poco::Util::AbstractConfiguration::Keys protocols;
+ config().keys("protocols", protocols);
+ for (const auto & protocol : protocols)
+ {
+ cert_path = config().getString("protocols." + protocol + ".certificateFile", "");
+ key_path = config().getString("protocols." + protocol + ".privateKeyFile", "");
+ if (!cert_path.empty())
+ extra_paths.emplace_back(cert_path);
+ if (!key_path.empty())
+ extra_paths.emplace_back(key_path);
+ }
+
auto main_config_reloader = std::make_unique(
config_path,
extra_paths,
@@ -1482,6 +1542,8 @@ try
global_context->setMaxDictionaryNumToWarn(new_server_settings.max_dictionary_num_to_warn);
global_context->setMaxDatabaseNumToWarn(new_server_settings.max_database_num_to_warn);
global_context->setMaxPartNumToWarn(new_server_settings.max_part_num_to_warn);
+ /// Only for system.server_settings
+ global_context->setConfigReloaderInterval(new_server_settings.config_reload_interval_ms);
SlotCount concurrent_threads_soft_limit = UnlimitedSlots;
if (new_server_settings.concurrent_threads_soft_limit_num > 0 && new_server_settings.concurrent_threads_soft_limit_num < concurrent_threads_soft_limit)
@@ -1571,6 +1633,10 @@ try
0, // We don't need any threads one all the parts will be deleted
new_server_settings.max_parts_cleaning_thread_pool_size);
+
+ global_context->setMergeWorkload(new_server_settings.merge_workload);
+ global_context->setMutationWorkload(new_server_settings.mutation_workload);
+
if (config->has("resources"))
{
global_context->getResourceManager()->updateConfiguration(*config);
@@ -1606,9 +1672,9 @@ try
CompressionCodecEncrypted::Configuration::instance().tryLoad(*config, "encryption_codecs");
#if USE_SSL
- CertificateReloader::instance().tryLoad(*config);
+ CertificateReloader::instance().tryReloadAll(*config);
#endif
- NamedCollectionUtils::reloadFromConfig(*config);
+ NamedCollectionFactory::instance().reloadFromConfig(*config);
FileCacheFactory::instance().updateSettingsFromConfig(*config);
@@ -1632,12 +1698,15 @@ try
if (global_context->isServerCompletelyStarted())
CannotAllocateThreadFaultInjector::setFaultProbability(new_server_settings.cannot_allocate_thread_fault_injection_probability);
+#if USE_GWP_ASAN
+ GWPAsan::setForceSampleProbability(new_server_settings.gwp_asan_force_sample_probability);
+#endif
+
ProfileEvents::increment(ProfileEvents::MainConfigLoads);
/// Must be the last.
latest_config = config;
- },
- /* already_loaded = */ false); /// Reload it right now (initial loading)
+ });
const auto listen_hosts = getListenHosts(config());
const auto interserver_listen_hosts = getInterserverListenHosts(config());
@@ -1750,11 +1819,6 @@ try
}
- if (config().has(DB::PlacementInfo::PLACEMENT_CONFIG_PREFIX))
- {
- PlacementInfo::PlacementInfo::instance().initialize(config());
- }
-
{
std::lock_guard lock(servers_lock);
/// We should start interserver communications before (and more important shutdown after) tables.
@@ -1889,6 +1953,7 @@ try
/// Set current database name before loading tables and databases because
/// system logs may copy global context.
+ std::string default_database = server_settings.default_database.toString();
global_context->setCurrentDatabaseNameInGlobalContext(default_database);
LOG_INFO(log, "Loading metadata from {}", path_str);
@@ -1950,52 +2015,9 @@ try
LOG_DEBUG(log, "Loaded metadata.");
- /// Init trace collector only after trace_log system table was created
- /// Disable it if we collect test coverage information, because it will work extremely slow.
-#if !WITH_COVERAGE
- /// Profilers cannot work reliably with any other libunwind or without PHDR cache.
- if (hasPHDRCache())
- {
+ if (has_trace_collector)
global_context->initializeTraceCollector();
- /// Set up server-wide memory profiler (for total memory tracker).
- if (server_settings.total_memory_profiler_step)
- {
- total_memory_tracker.setProfilerStep(server_settings.total_memory_profiler_step);
- }
-
- if (server_settings.total_memory_tracker_sample_probability > 0.0)
- {
- total_memory_tracker.setSampleProbability(server_settings.total_memory_tracker_sample_probability);
- }
-
- if (server_settings.total_memory_profiler_sample_min_allocation_size)
- {
- total_memory_tracker.setSampleMinAllocationSize(server_settings.total_memory_profiler_sample_min_allocation_size);
- }
-
- if (server_settings.total_memory_profiler_sample_max_allocation_size)
- {
- total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size);
- }
- }
-#endif
-
- /// Describe multiple reasons when query profiler cannot work.
-
-#if WITH_COVERAGE
- LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they work extremely slow with test coverage.");
-#endif
-
-#if defined(SANITIZER)
- LOG_INFO(log, "Query Profiler disabled because they cannot work under sanitizers"
- " when two different stack unwinding methods will interfere with each other.");
-#endif
-
- if (!hasPHDRCache())
- LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created"
- " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe).");
-
#if defined(OS_LINUX)
auto tasks_stats_provider = TasksStatsCounters::findBestAvailableProvider();
if (tasks_stats_provider == TasksStatsCounters::MetricsProvider::None)
@@ -2124,6 +2146,10 @@ try
CannotAllocateThreadFaultInjector::setFaultProbability(server_settings.cannot_allocate_thread_fault_injection_probability);
+#if USE_GWP_ASAN
+ GWPAsan::setForceSampleProbability(server_settings.gwp_asan_force_sample_probability);
+#endif
+
try
{
global_context->startClusterDiscovery();
diff --git a/programs/server/config.xml b/programs/server/config.xml
index 4b3248d9d1c..94825a55f67 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -29,7 +29,14 @@
-->
1000M
10
+
+
+
+
+
+
+
-
+ true
@@ -408,13 +415,11 @@
- 5368709120
+ You should not lower this value. -->
+
-
- 5368709120
+
+
- 1000
+
- 134217728
+
- 10000
+
+
+
+
/var/lib/clickhouse/caches/
@@ -1155,6 +1170,18 @@
false
+
+
+ system
+
+ 7500
+ 1048576
+ 8192
+ 524288
+ 1000
+ false
+
+
+
+
+
-
-
- 1073741824
- 1024
- 1048576
- 30000000
-
-
backups
diff --git a/programs/server/config.yaml.example b/programs/server/config.yaml.example
index 9fc188e97aa..5d5499f876c 100644
--- a/programs/server/config.yaml.example
+++ b/programs/server/config.yaml.example
@@ -260,7 +260,10 @@ uncompressed_cache_size: 8589934592
# Approximate size of mark cache, used in tables of MergeTree family.
# In bytes. Cache is single for server. Memory is allocated only on demand.
# You should not lower this value.
-mark_cache_size: 5368709120
+# mark_cache_size: 5368709120
+
+# For marks of secondary indices.
+# index_mark_cache_size: 5368709120
# If you enable the `min_bytes_to_use_mmap_io` setting,
# the data in MergeTree tables can be read with mmap to avoid copying from kernel to userspace.
@@ -277,13 +280,20 @@ mark_cache_size: 5368709120
# in query or server memory usage - because this memory can be discarded similar to OS page cache.
# The cache is dropped (the files are closed) automatically on removal of old parts in MergeTree,
# also it can be dropped manually by the SYSTEM DROP MMAP CACHE query.
-mmap_cache_size: 1000
+# mmap_cache_size: 1024
# Cache size in bytes for compiled expressions.
-compiled_expression_cache_size: 134217728
+# compiled_expression_cache_size: 134217728
# Cache size in elements for compiled expressions.
-compiled_expression_cache_elements_size: 10000
+# compiled_expression_cache_elements_size: 10000
+
+# Configuration for the query cache
+# query_cache:
+# max_size_in_bytes: 1073741824
+# max_entries: 1024
+# max_entry_size_in_bytes: 1048576
+# max_entry_size_in_rows: 30000000
# Path to data directory, with trailing slash.
path: /var/lib/clickhouse/
@@ -726,6 +736,13 @@ metric_log:
flush_interval_milliseconds: 7500
collect_interval_milliseconds: 1000
+# Error log contains rows with current values of errors collected with "collect_interval_milliseconds" interval.
+error_log:
+ database: system
+ table: error_log
+ flush_interval_milliseconds: 7500
+ collect_interval_milliseconds: 1000
+
# Asynchronous metric log contains values of metrics from
# system.asynchronous_metrics.
asynchronous_metric_log:
diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp
index c3bb42160ad..353358fac65 100644
--- a/src/Access/AccessControl.cpp
+++ b/src/Access/AccessControl.cpp
@@ -261,7 +261,24 @@ AccessControl::AccessControl()
}
-AccessControl::~AccessControl() = default;
+AccessControl::~AccessControl()
+{
+ try
+ {
+ AccessControl::shutdown();
+ }
+ catch (...)
+ {
+ tryLogCurrentException(__PRETTY_FUNCTION__);
+ }
+}
+
+
+void AccessControl::shutdown()
+{
+ MultipleAccessStorage::shutdown();
+ removeAllStorages();
+}
void AccessControl::setUpFromMainConfig(const Poco::Util::AbstractConfiguration & config_, const String & config_path_,
diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h
index d1537219a06..bfaf256ad48 100644
--- a/src/Access/AccessControl.h
+++ b/src/Access/AccessControl.h
@@ -53,6 +53,9 @@ public:
AccessControl();
~AccessControl() override;
+ /// Shutdown the access control and stops all background activity.
+ void shutdown() override;
+
/// Initializes access storage (user directories).
void setUpFromMainConfig(const Poco::Util::AbstractConfiguration & config_, const String & config_path_,
const zkutil::GetZooKeeper & get_zookeeper_function_);
diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp
index bf1fe3feec3..6b9a6e05cf6 100644
--- a/src/Access/Authentication.cpp
+++ b/src/Access/Authentication.cpp
@@ -8,6 +8,7 @@
#include
#include
#include
+#include
#include "config.h"
@@ -108,6 +109,9 @@ bool Authentication::areCredentialsValid(
case AuthenticationType::HTTP:
throw Authentication::Require("ClickHouse Basic Authentication");
+ case AuthenticationType::JWT:
+ throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+
case AuthenticationType::KERBEROS:
return external_authenticators.checkKerberosCredentials(auth_data.getKerberosRealm(), *gss_acceptor_context);
@@ -149,6 +153,9 @@ bool Authentication::areCredentialsValid(
case AuthenticationType::SSL_CERTIFICATE:
throw Authentication::Require("ClickHouse X.509 Authentication");
+ case AuthenticationType::JWT:
+ throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+
case AuthenticationType::SSH_KEY:
#if USE_SSH
throw Authentication::Require("SSH Keys Authentication");
@@ -193,6 +200,9 @@ bool Authentication::areCredentialsValid(
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
#endif
+ case AuthenticationType::JWT:
+ throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+
case AuthenticationType::BCRYPT_PASSWORD:
return checkPasswordBcrypt(basic_credentials->getPassword(), auth_data.getPasswordHashBinary());
@@ -222,11 +232,22 @@ bool Authentication::areCredentialsValid(
case AuthenticationType::HTTP:
throw Authentication::Require("ClickHouse Basic Authentication");
+ case AuthenticationType::JWT:
+ throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+
case AuthenticationType::KERBEROS:
throw Authentication::Require(auth_data.getKerberosRealm());
case AuthenticationType::SSL_CERTIFICATE:
- return auth_data.getSSLCertificateCommonNames().contains(ssl_certificate_credentials->getCommonName());
+ for (SSLCertificateSubjects::Type type : {SSLCertificateSubjects::Type::CN, SSLCertificateSubjects::Type::SAN})
+ {
+ for (const auto & subject : auth_data.getSSLCertificateSubjects().at(type))
+ {
+ if (ssl_certificate_credentials->getSSLCertificateSubjects().at(type).contains(subject))
+ return true;
+ }
+ }
+ return false;
case AuthenticationType::SSH_KEY:
#if USE_SSH
@@ -254,6 +275,9 @@ bool Authentication::areCredentialsValid(
case AuthenticationType::HTTP:
throw Authentication::Require("ClickHouse Basic Authentication");
+ case AuthenticationType::JWT:
+ throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+
case AuthenticationType::KERBEROS:
throw Authentication::Require(auth_data.getKerberosRealm());
diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp
index a32215f3d92..5a35eeefe5b 100644
--- a/src/Access/AuthenticationData.cpp
+++ b/src/Access/AuthenticationData.cpp
@@ -15,6 +15,7 @@
#include
#include
+#include
#include "config.h"
#if USE_SSL
@@ -31,6 +32,7 @@ namespace DB
{
namespace ErrorCodes
{
+ extern const int AUTHENTICATION_FAILED;
extern const int SUPPORT_IS_DISABLED;
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
@@ -90,8 +92,10 @@ bool AuthenticationData::Util::checkPasswordBcrypt(std::string_view password [[m
{
#if USE_BCRYPT
int ret = bcrypt_checkpw(password.data(), reinterpret_cast(password_bcrypt.data()));
+ /// Before 24.6 we didn't validate hashes on creation, so it could be that the stored hash is invalid
+ /// and it could not be decoded by the library
if (ret == -1)
- throw Exception(ErrorCodes::LOGICAL_ERROR, "BCrypt library failed: bcrypt_checkpw returned {}", ret);
+ throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Internal failure decoding Bcrypt hash");
return (ret == 0);
#else
throw Exception(
@@ -104,7 +108,7 @@ bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs)
{
return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash)
&& (lhs.ldap_server_name == rhs.ldap_server_name) && (lhs.kerberos_realm == rhs.kerberos_realm)
- && (lhs.ssl_certificate_common_names == rhs.ssl_certificate_common_names)
+ && (lhs.ssl_certificate_subjects == rhs.ssl_certificate_subjects)
#if USE_SSH
&& (lhs.ssh_keys == rhs.ssh_keys)
#endif
@@ -132,6 +136,7 @@ void AuthenticationData::setPassword(const String & password_)
case AuthenticationType::BCRYPT_PASSWORD:
case AuthenticationType::NO_PASSWORD:
case AuthenticationType::LDAP:
+ case AuthenticationType::JWT:
case AuthenticationType::KERBEROS:
case AuthenticationType::SSL_CERTIFICATE:
case AuthenticationType::SSH_KEY:
@@ -230,6 +235,17 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Password hash for the 'BCRYPT_PASSWORD' authentication type has length {} "
"but must be 59 or 60 bytes.", hash.size());
+
+ auto resized = hash;
+ resized.resize(64);
+
+#if USE_BCRYPT
+ /// Verify that it is a valid hash
+ int ret = bcrypt_checkpw("", reinterpret_cast(resized.data()));
+ if (ret == -1)
+ throw Exception(ErrorCodes::BAD_ARGUMENTS, "Could not decode the provided hash with 'bcrypt_hash'");
+#endif
+
password_hash = hash;
password_hash.resize(64);
return;
@@ -237,6 +253,7 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
case AuthenticationType::NO_PASSWORD:
case AuthenticationType::LDAP:
+ case AuthenticationType::JWT:
case AuthenticationType::KERBEROS:
case AuthenticationType::SSL_CERTIFICATE:
case AuthenticationType::SSH_KEY:
@@ -261,11 +278,16 @@ String AuthenticationData::getSalt() const
return salt;
}
-void AuthenticationData::setSSLCertificateCommonNames(boost::container::flat_set common_names_)
+void AuthenticationData::setSSLCertificateSubjects(SSLCertificateSubjects && ssl_certificate_subjects_)
{
- if (common_names_.empty())
- throw Exception(ErrorCodes::BAD_ARGUMENTS, "The 'SSL CERTIFICATE' authentication type requires a non-empty list of common names.");
- ssl_certificate_common_names = std::move(common_names_);
+ if (ssl_certificate_subjects_.empty())
+ throw Exception(ErrorCodes::BAD_ARGUMENTS, "The 'SSL CERTIFICATE' authentication type requires a non-empty list of subjects.");
+ ssl_certificate_subjects = std::move(ssl_certificate_subjects_);
+}
+
+void AuthenticationData::addSSLCertificateSubject(SSLCertificateSubjects::Type type_, String && subject_)
+{
+ ssl_certificate_subjects.insert(type_, std::move(subject_));
}
std::shared_ptr AuthenticationData::toAST() const
@@ -308,6 +330,10 @@ std::shared_ptr AuthenticationData::toAST() const
node->children.push_back(std::make_shared(getLDAPServerName()));
break;
}
+ case AuthenticationType::JWT:
+ {
+ throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+ }
case AuthenticationType::KERBEROS:
{
const auto & realm = getKerberosRealm();
@@ -319,7 +345,14 @@ std::shared_ptr AuthenticationData::toAST() const
}
case AuthenticationType::SSL_CERTIFICATE:
{
- for (const auto & name : getSSLCertificateCommonNames())
+ using SSLCertificateSubjects::Type::CN;
+ using SSLCertificateSubjects::Type::SAN;
+
+ const auto &subjects = getSSLCertificateSubjects();
+ SSLCertificateSubjects::Type cert_subject_type = !subjects.at(SAN).empty() ? SAN : CN;
+
+ node->ssl_cert_subject_type = toString(cert_subject_type);
+ for (const auto & name : getSSLCertificateSubjects().at(cert_subject_type))
node->children.push_back(std::make_shared(name));
break;
@@ -493,11 +526,9 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
}
else if (query.type == AuthenticationType::SSL_CERTIFICATE)
{
- boost::container::flat_set common_names;
+ auto ssl_cert_subject_type = parseSSLCertificateSubjectType(*query.ssl_cert_subject_type);
for (const auto & arg : args)
- common_names.insert(checkAndGetLiteralArgument(arg, "common_name"));
-
- auth_data.setSSLCertificateCommonNames(std::move(common_names));
+ auth_data.addSSLCertificateSubject(ssl_cert_subject_type, checkAndGetLiteralArgument(arg, "ssl_certificate_subject"));
}
else if (query.type == AuthenticationType::HTTP)
{
diff --git a/src/Access/AuthenticationData.h b/src/Access/AuthenticationData.h
index c97e0327b56..8093fe1d888 100644
--- a/src/Access/AuthenticationData.h
+++ b/src/Access/AuthenticationData.h
@@ -2,13 +2,14 @@
#include
#include
+#include
#include
#include
#include
#include
#include
-#include
+
#include "config.h"
@@ -58,8 +59,9 @@ public:
const String & getKerberosRealm() const { return kerberos_realm; }
void setKerberosRealm(const String & realm) { kerberos_realm = realm; }
- const boost::container::flat_set & getSSLCertificateCommonNames() const { return ssl_certificate_common_names; }
- void setSSLCertificateCommonNames(boost::container::flat_set common_names_);
+ const SSLCertificateSubjects & getSSLCertificateSubjects() const { return ssl_certificate_subjects; }
+ void setSSLCertificateSubjects(SSLCertificateSubjects && ssl_certificate_subjects_);
+ void addSSLCertificateSubject(SSLCertificateSubjects::Type type_, String && subject_);
#if USE_SSH
const std::vector & getSSHKeys() const { return ssh_keys; }
@@ -96,7 +98,7 @@ private:
Digest password_hash;
String ldap_server_name;
String kerberos_realm;
- boost::container::flat_set ssl_certificate_common_names;
+ SSLCertificateSubjects ssl_certificate_subjects;
String salt;
#if USE_SSH
std::vector ssh_keys;
diff --git a/src/Access/CachedAccessChecking.cpp b/src/Access/CachedAccessChecking.cpp
index aa8ef6073d3..0d629e7b77a 100644
--- a/src/Access/CachedAccessChecking.cpp
+++ b/src/Access/CachedAccessChecking.cpp
@@ -4,12 +4,12 @@
namespace DB
{
-CachedAccessChecking::CachedAccessChecking(const std::shared_ptr & access_, AccessFlags access_flags_)
+CachedAccessChecking::CachedAccessChecking(const std::shared_ptr & access_, AccessFlags access_flags_)
: CachedAccessChecking(access_, AccessRightsElement{access_flags_})
{
}
-CachedAccessChecking::CachedAccessChecking(const std::shared_ptr & access_, const AccessRightsElement & element_)
+CachedAccessChecking::CachedAccessChecking(const std::shared_ptr & access_, const AccessRightsElement & element_)
: access(access_), element(element_)
{
}
diff --git a/src/Access/CachedAccessChecking.h b/src/Access/CachedAccessChecking.h
index e87c28dd823..aaeea6ceddc 100644
--- a/src/Access/CachedAccessChecking.h
+++ b/src/Access/CachedAccessChecking.h
@@ -1,6 +1,7 @@
#pragma once
#include
+#include
#include
@@ -13,14 +14,14 @@ class ContextAccess;
class CachedAccessChecking
{
public:
- CachedAccessChecking(const std::shared_ptr & access_, AccessFlags access_flags_);
- CachedAccessChecking(const std::shared_ptr & access_, const AccessRightsElement & element_);
+ CachedAccessChecking(const std::shared_ptr & access_, AccessFlags access_flags_);
+ CachedAccessChecking(const std::shared_ptr & access_, const AccessRightsElement & element_);
~CachedAccessChecking();
bool checkAccess(bool throw_if_denied = true);
private:
- const std::shared_ptr access;
+ const std::shared_ptr access;
const AccessRightsElement element;
bool checked = false;
bool result = false;
diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h
index 7f0eff2184b..e9f24a8c685 100644
--- a/src/Access/Common/AccessType.h
+++ b/src/Access/Common/AccessType.h
@@ -51,10 +51,11 @@ enum class AccessType : uint8_t
M(ALTER_CLEAR_INDEX, "CLEAR INDEX", TABLE, ALTER_INDEX) \
M(ALTER_INDEX, "INDEX", GROUP, ALTER_TABLE) /* allows to execute ALTER ORDER BY or ALTER {ADD|DROP...} INDEX */\
\
- M(ALTER_ADD_STATISTIC, "ALTER ADD STATISTIC", TABLE, ALTER_STATISTIC) \
- M(ALTER_DROP_STATISTIC, "ALTER DROP STATISTIC", TABLE, ALTER_STATISTIC) \
- M(ALTER_MATERIALIZE_STATISTIC, "ALTER MATERIALIZE STATISTIC", TABLE, ALTER_STATISTIC) \
- M(ALTER_STATISTIC, "STATISTIC", GROUP, ALTER_TABLE) /* allows to execute ALTER STATISTIC */\
+ M(ALTER_ADD_STATISTICS, "ALTER ADD STATISTIC", TABLE, ALTER_STATISTICS) \
+ M(ALTER_DROP_STATISTICS, "ALTER DROP STATISTIC", TABLE, ALTER_STATISTICS) \
+ M(ALTER_MODIFY_STATISTICS, "ALTER MODIFY STATISTIC", TABLE, ALTER_STATISTICS) \
+ M(ALTER_MATERIALIZE_STATISTICS, "ALTER MATERIALIZE STATISTIC", TABLE, ALTER_STATISTICS) \
+ M(ALTER_STATISTICS, "STATISTIC", GROUP, ALTER_TABLE) /* allows to execute ALTER STATISTIC */\
\
M(ALTER_ADD_PROJECTION, "ADD PROJECTION", TABLE, ALTER_PROJECTION) \
M(ALTER_DROP_PROJECTION, "DROP PROJECTION", TABLE, ALTER_PROJECTION) \
diff --git a/src/Access/Common/AuthenticationType.cpp b/src/Access/Common/AuthenticationType.cpp
index 2cc126ad9b7..427765b8a79 100644
--- a/src/Access/Common/AuthenticationType.cpp
+++ b/src/Access/Common/AuthenticationType.cpp
@@ -72,6 +72,11 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty
static const auto info = make_info(Keyword::HTTP);
return info;
}
+ case AuthenticationType::JWT:
+ {
+ static const auto info = make_info(Keyword::JWT);
+ return info;
+ }
case AuthenticationType::MAX:
break;
}
diff --git a/src/Access/Common/AuthenticationType.h b/src/Access/Common/AuthenticationType.h
index a68549aff4c..16f4388bbff 100644
--- a/src/Access/Common/AuthenticationType.h
+++ b/src/Access/Common/AuthenticationType.h
@@ -41,6 +41,9 @@ enum class AuthenticationType : uint8_t
/// Authentication through HTTP protocol
HTTP,
+ /// JSON Web Token
+ JWT,
+
MAX,
};
diff --git a/src/Access/Common/SSLCertificateSubjects.cpp b/src/Access/Common/SSLCertificateSubjects.cpp
new file mode 100644
index 00000000000..ca7001a31a2
--- /dev/null
+++ b/src/Access/Common/SSLCertificateSubjects.cpp
@@ -0,0 +1,95 @@
+#include
+#include
+
+#if USE_SSL
+#include
+#endif
+
+namespace DB
+{
+namespace ErrorCodes
+{
+ extern const int BAD_ARGUMENTS;
+}
+
+#if USE_SSL
+SSLCertificateSubjects extractSSLCertificateSubjects(const Poco::Net::X509Certificate & certificate)
+{
+
+ SSLCertificateSubjects subjects;
+ if (!certificate.commonName().empty())
+ {
+ subjects.insert(SSLCertificateSubjects::Type::CN, certificate.commonName());
+ }
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wused-but-marked-unused"
+ auto stackof_general_name_deleter = [](void * ptr) { GENERAL_NAMES_free(static_cast(ptr)); };
+ std::unique_ptr cert_names(
+ X509_get_ext_d2i(const_cast(certificate.certificate()), NID_subject_alt_name, nullptr, nullptr),
+ stackof_general_name_deleter);
+
+ if (STACK_OF(GENERAL_NAME) * names = static_cast(cert_names.get()))
+ {
+ for (int i = 0; i < sk_GENERAL_NAME_num(names); ++i)
+ {
+ const GENERAL_NAME * name = sk_GENERAL_NAME_value(names, i);
+ if (name->type == GEN_DNS || name->type == GEN_URI)
+ {
+ const char * data = reinterpret_cast(ASN1_STRING_get0_data(name->d.ia5));
+ std::size_t len = ASN1_STRING_length(name->d.ia5);
+ std::string subject = (name->type == GEN_DNS ? "DNS:" : "URI:") + std::string(data, len);
+ subjects.insert(SSLCertificateSubjects::Type::SAN, std::move(subject));
+ }
+ }
+ }
+
+#pragma clang diagnostic pop
+ return subjects;
+}
+#endif
+
+
+void SSLCertificateSubjects::insert(const String & subject_type_, String && subject)
+{
+ insert(parseSSLCertificateSubjectType(subject_type_), std::move(subject));
+}
+
+void SSLCertificateSubjects::insert(Type subject_type_, String && subject)
+{
+ subjects[static_cast(subject_type_)].insert(std::move(subject));
+}
+
+SSLCertificateSubjects::Type parseSSLCertificateSubjectType(const String & type_)
+{
+ if (type_ == "CN")
+ return SSLCertificateSubjects::Type::CN;
+ if (type_ == "SAN")
+ return SSLCertificateSubjects::Type::SAN;
+
+ throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown SSL Certificate Subject Type: {}", type_);
+}
+
+String toString(SSLCertificateSubjects::Type type_)
+{
+ switch (type_)
+ {
+ case SSLCertificateSubjects::Type::CN:
+ return "CN";
+ case SSLCertificateSubjects::Type::SAN:
+ return "SAN";
+ }
+}
+
+bool operator==(const SSLCertificateSubjects & lhs, const SSLCertificateSubjects & rhs)
+{
+ for (SSLCertificateSubjects::Type type : {SSLCertificateSubjects::Type::CN, SSLCertificateSubjects::Type::SAN})
+ {
+ if (lhs.at(type) != rhs.at(type))
+ return false;
+ }
+ return true;
+}
+
+}
+
diff --git a/src/Access/Common/SSLCertificateSubjects.h b/src/Access/Common/SSLCertificateSubjects.h
new file mode 100644
index 00000000000..ec11714d48a
--- /dev/null
+++ b/src/Access/Common/SSLCertificateSubjects.h
@@ -0,0 +1,48 @@
+#pragma once
+
+#include "config.h"
+#include
+#include
+
+#if USE_SSL
+# include
+#endif
+
+namespace DB
+{
+class SSLCertificateSubjects
+{
+public:
+ using container = boost::container::flat_set;
+ enum class Type
+ {
+ CN,
+ SAN
+ };
+
+private:
+ std::array subjects;
+
+public:
+ inline const container & at(Type type_) const { return subjects[static_cast(type_)]; }
+ inline bool empty()
+ {
+ for (auto & subject_list : subjects)
+ {
+ if (!subject_list.empty())
+ return false;
+ }
+ return true;
+ }
+ void insert(const String & subject_type_, String && subject);
+ void insert(Type type_, String && subject);
+ friend bool operator==(const SSLCertificateSubjects & lhs, const SSLCertificateSubjects & rhs);
+};
+
+String toString(SSLCertificateSubjects::Type type_);
+SSLCertificateSubjects::Type parseSSLCertificateSubjectType(const String & type_);
+
+#if USE_SSL
+SSLCertificateSubjects extractSSLCertificateSubjects(const Poco::Net::X509Certificate & certificate);
+#endif
+}
diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp
index 2a658d7aaa2..a2807ecc5ea 100644
--- a/src/Access/ContextAccess.cpp
+++ b/src/Access/ContextAccess.cpp
@@ -20,6 +20,7 @@
#include
#include
#include
+#include
namespace DB
@@ -271,7 +272,7 @@ namespace
std::shared_ptr ContextAccess::fromContext(const ContextPtr & context)
{
- return context->getAccess();
+ return ContextAccessWrapper::fromContext(context)->getAccess();
}
@@ -360,10 +361,13 @@ void ContextAccess::setUser(const UserPtr & user_) const
subscription_for_roles_changes.reset();
enabled_roles = access_control->getEnabledRoles(current_roles, current_roles_with_admin_option);
- subscription_for_roles_changes = enabled_roles->subscribeForChanges([this](const std::shared_ptr & roles_info_)
+ subscription_for_roles_changes = enabled_roles->subscribeForChanges([weak_ptr = weak_from_this()](const std::shared_ptr & roles_info_)
{
- std::lock_guard lock{mutex};
- setRolesInfo(roles_info_);
+ auto ptr = weak_ptr.lock();
+ if (!ptr)
+ return;
+ std::lock_guard lock{ptr->mutex};
+ ptr->setRolesInfo(roles_info_);
});
setRolesInfo(enabled_roles->getRolesInfo());
@@ -557,7 +561,7 @@ std::shared_ptr ContextAccess::getAccessRightsWithImplicit()
template
-bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... args) const
+bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlags flags, const Args &... args) const
{
if (user_was_dropped)
{
@@ -570,8 +574,10 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
if (params.full_access)
return true;
- auto access_granted = []
+ auto access_granted = [&]
{
+ if constexpr (throw_if_denied)
+ context->addQueryPrivilegesInfo(AccessRightsElement{flags, args...}.toStringWithoutOptions(), true);
return true;
};
@@ -580,7 +586,10 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
FmtArgs && ...fmt_args [[maybe_unused]])
{
if constexpr (throw_if_denied)
+ {
+ context->addQueryPrivilegesInfo(AccessRightsElement{flags, args...}.toStringWithoutOptions(), false);
throw Exception(error_code, std::move(fmt_string), getUserName(), std::forward(fmt_args)...);
+ }
return false;
};
@@ -683,102 +692,102 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
}
template
-bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const
+bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessFlags & flags) const
{
- return checkAccessImplHelper(flags);
+ return checkAccessImplHelper(context, flags);
}
template
-bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const
+bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessFlags & flags, std::string_view database, const Args &... args) const
{
- return checkAccessImplHelper(flags, database.empty() ? params.current_database : database, args...);
+ return checkAccessImplHelper(context, flags, database.empty() ? params.current_database : database, args...);
}
template
-bool ContextAccess::checkAccessImplHelper(const AccessRightsElement & element) const
+bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, const AccessRightsElement & element) const
{
assert(!element.grant_option || grant_option);
if (element.isGlobalWithParameter())
{
if (element.any_parameter)
- return checkAccessImpl(element.access_flags);
+ return checkAccessImpl(context, element.access_flags);
else
- return checkAccessImpl(element.access_flags, element.parameter);
+ return checkAccessImpl(context, element.access_flags, element.parameter);
}
else if (element.any_database)
- return checkAccessImpl(element.access_flags);
+ return checkAccessImpl(context, element.access_flags);
else if (element.any_table)
- return checkAccessImpl(element.access_flags, element.database);
+ return checkAccessImpl(context, element.access_flags, element.database);
else if (element.any_column)
- return checkAccessImpl(element.access_flags, element.database, element.table);
+ return checkAccessImpl(context, element.access_flags, element.database, element.table);
else
- return checkAccessImpl(element.access_flags, element.database, element.table, element.columns);
+ return checkAccessImpl(context, element.access_flags, element.database, element.table, element.columns);
}
template
-bool ContextAccess::checkAccessImpl(const AccessRightsElement & element) const
+bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessRightsElement & element) const
{
if constexpr (grant_option)
{
- return checkAccessImplHelper(element);
+ return checkAccessImplHelper(context, element);
}
else
{
if (element.grant_option)
- return checkAccessImplHelper(element);
+ return checkAccessImplHelper(context, element);
else
- return checkAccessImplHelper(element);
+ return checkAccessImplHelper(context, element);
}
}
template
-bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
+bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessRightsElements & elements) const
{
for (const auto & element : elements)
- if (!checkAccessImpl(element))
+ if (!checkAccessImpl(context, element))
return false;
return true;
}
-bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl(flags); }
-bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl(flags, database); }
-bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl(flags, database, table); }
-bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl(flags, database, table, column); }
-bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const { return checkAccessImpl(flags, database, table, columns); }
-bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl(flags, database, table, columns); }
-bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl(element); }
-bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl(elements); }
+bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl(context, flags); }
+bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl(context, flags, database); }
+bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl(context, flags, database, table); }
+bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl(context, flags, database, table, column); }
+bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const { return checkAccessImpl