diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index f0741b5465f..e5b797beebd 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -850,6 +850,48 @@ jobs:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+ BuilderBinRISCV64:
+ needs: [DockerHubPush]
+ runs-on: [self-hosted, builder]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/build_check
+ IMAGES_PATH=${{runner.temp}}/images_path
+ REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+ CACHES_PATH=${{runner.temp}}/../ccaches
+ BUILD_NAME=binary_riscv64
+ EOF
+ - name: Download changed images
+ uses: actions/download-artifact@v3
+ with:
+ name: changed_images
+ path: ${{ env.IMAGES_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ submodules: true
+ fetch-depth: 0 # otherwise we will have no info about contributors
+ - name: Build
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+ - name: Upload build URLs to artifacts
+ if: ${{ success() || failure() }}
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ env.BUILD_URLS }}
+ path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
############################################################################################
##################################### Docker images #######################################
############################################################################################
@@ -932,6 +974,7 @@ jobs:
- BuilderBinDarwinAarch64
- BuilderBinFreeBSD
- BuilderBinPPC64
+ - BuilderBinRISCV64
- BuilderBinAmd64Compat
- BuilderBinAarch64V80Compat
- BuilderBinClangTidy
@@ -2827,6 +2870,216 @@ jobs:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan0:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=0
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan1:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=1
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan2:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=2
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan3:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=3
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan4:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=4
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan5:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=5
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan0:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]
@@ -3920,6 +4173,12 @@ jobs:
- IntegrationTestsAsan3
- IntegrationTestsAsan4
- IntegrationTestsAsan5
+ - IntegrationTestsAnalyzerAsan0
+ - IntegrationTestsAnalyzerAsan1
+ - IntegrationTestsAnalyzerAsan2
+ - IntegrationTestsAnalyzerAsan3
+ - IntegrationTestsAnalyzerAsan4
+ - IntegrationTestsAnalyzerAsan5
- IntegrationTestsRelease0
- IntegrationTestsRelease1
- IntegrationTestsRelease2
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index acf6bbe8f6a..9de0444bd83 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -75,54 +75,7 @@ jobs:
Codebrowser:
needs: [DockerHubPush]
uses: ./.github/workflows/woboq.yml
- BuilderCoverity:
- needs: DockerHubPush
- runs-on: [self-hosted, builder]
- steps:
- - name: Set envs
- run: |
- cat >> "$GITHUB_ENV" << 'EOF'
- BUILD_NAME=coverity
- CACHES_PATH=${{runner.temp}}/../ccaches
- IMAGES_PATH=${{runner.temp}}/images_path
- REPO_COPY=${{runner.temp}}/build_check/ClickHouse
- TEMP_PATH=${{runner.temp}}/build_check
- EOF
- echo "COVERITY_TOKEN=${{ secrets.COVERITY_TOKEN }}" >> "$GITHUB_ENV"
- - name: Download changed images
- uses: actions/download-artifact@v3
- with:
- name: changed_images
- path: ${{ env.IMAGES_PATH }}
- - name: Check out repository code
- uses: ClickHouse/checkout@v1
- with:
- clear-repository: true
- submodules: true
- - name: Build
- run: |
- sudo rm -fr "$TEMP_PATH"
- mkdir -p "$TEMP_PATH"
- cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
- cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
- - name: Upload Coverity Analysis
- if: ${{ success() || failure() }}
- run: |
- curl --form token="${COVERITY_TOKEN}" \
- --form email='security+coverity@clickhouse.com' \
- --form file="@$TEMP_PATH/$BUILD_NAME/coverity-scan.tar.gz" \
- --form version="${GITHUB_REF#refs/heads/}-${GITHUB_SHA::6}" \
- --form description="Nighly Scan: $(date +'%Y-%m-%dT%H:%M:%S')" \
- https://scan.coverity.com/builds?project=ClickHouse%2FClickHouse
- - name: Cleanup
- if: always()
- run: |
- docker ps --quiet | xargs --no-run-if-empty docker kill ||:
- docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
- sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
SonarCloud:
- # TODO: Remove if: whenever SonarCloud supports c++23
- if: ${{ false }}
runs-on: [self-hosted, builder]
env:
SONAR_SCANNER_VERSION: 4.8.0.2856
@@ -159,7 +112,7 @@ jobs:
- name: Set Up Build Tools
run: |
sudo apt-get update
- sudo apt-get install -yq git cmake ccache ninja-build python3 yasm
+ sudo apt-get install -yq git cmake ccache ninja-build python3 yasm nasm
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
- name: Run build-wrapper
run: |
@@ -178,4 +131,5 @@ jobs:
--define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" \
--define sonar.projectKey="ClickHouse_ClickHouse" \
--define sonar.organization="clickhouse-java" \
- --define sonar.exclusions="**/*.java,**/*.ts,**/*.js,**/*.css,**/*.sql" \
+ --define sonar.cfamily.cpp23.enabled=true \
+ --define sonar.exclusions="**/*.java,**/*.ts,**/*.js,**/*.css,**/*.sql"
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index afc08f3e637..dd834959578 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -911,6 +911,47 @@ jobs:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+ BuilderBinRISCV64:
+ needs: [DockerHubPush, FastTest, StyleCheck]
+ runs-on: [self-hosted, builder]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/build_check
+ IMAGES_PATH=${{runner.temp}}/images_path
+ REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+ CACHES_PATH=${{runner.temp}}/../ccaches
+ BUILD_NAME=binary_riscv64
+ EOF
+ - name: Download changed images
+ uses: actions/download-artifact@v3
+ with:
+ name: changed_images
+ path: ${{ env.IMAGES_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ submodules: true
+ - name: Build
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+ - name: Upload build URLs to artifacts
+ if: ${{ success() || failure() }}
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ env.BUILD_URLS }}
+ path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
############################################################################################
##################################### Docker images #######################################
############################################################################################
@@ -992,6 +1033,7 @@ jobs:
- BuilderBinDarwinAarch64
- BuilderBinFreeBSD
- BuilderBinPPC64
+ - BuilderBinRISCV64
- BuilderBinAmd64Compat
- BuilderBinAarch64V80Compat
- BuilderBinClangTidy
@@ -3861,6 +3903,216 @@ jobs:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan0:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=0
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan1:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=1
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan2:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=2
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan3:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=3
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan4:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=4
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
+ IntegrationTestsAnalyzerAsan5:
+ needs: [BuilderDebAsan]
+ runs-on: [self-hosted, stress-tester]
+ steps:
+ - name: Set envs
+ run: |
+ cat >> "$GITHUB_ENV" << 'EOF'
+ TEMP_PATH=${{runner.temp}}/integration_tests_asan
+ REPORTS_PATH=${{runner.temp}}/reports_dir
+ CHECK_NAME=Integration tests (asan, analyzer)
+ REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
+ RUN_BY_HASH_NUM=5
+ RUN_BY_HASH_TOTAL=6
+ EOF
+ - name: Download json reports
+ uses: actions/download-artifact@v3
+ with:
+ path: ${{ env.REPORTS_PATH }}
+ - name: Check out repository code
+ uses: ClickHouse/checkout@v1
+ with:
+ clear-repository: true
+ - name: Integration test
+ run: |
+ sudo rm -fr "$TEMP_PATH"
+ mkdir -p "$TEMP_PATH"
+ cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+ cd "$REPO_COPY/tests/ci"
+ python3 integration_test_check.py "$CHECK_NAME"
+ - name: Cleanup
+ if: always()
+ run: |
+ docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+ docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+ sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan0:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]
@@ -4847,6 +5099,12 @@ jobs:
- IntegrationTestsAsan3
- IntegrationTestsAsan4
- IntegrationTestsAsan5
+ - IntegrationTestsAnalyzerAsan0
+ - IntegrationTestsAnalyzerAsan1
+ - IntegrationTestsAnalyzerAsan2
+ - IntegrationTestsAnalyzerAsan3
+ - IntegrationTestsAnalyzerAsan4
+ - IntegrationTestsAnalyzerAsan5
- IntegrationTestsRelease0
- IntegrationTestsRelease1
- IntegrationTestsRelease2
diff --git a/.github/workflows/woboq.yml b/.github/workflows/woboq.yml
index bdfbc8fef9c..1ef729af30a 100644
--- a/.github/workflows/woboq.yml
+++ b/.github/workflows/woboq.yml
@@ -12,6 +12,7 @@ jobs:
# don't use dockerhub push because this image updates so rarely
WoboqCodebrowser:
runs-on: [self-hosted, style-checker]
+ timeout-minutes: 420 # the task is pretty heavy, so there's an additional hour
steps:
- name: Set envs
run: |
diff --git a/.gitignore b/.gitignore
index a04c60d5ca3..5341f23a94f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -69,6 +69,7 @@ cmake-build-*
*.pyc
__pycache__
*.pytest_cache
+.mypy_cache
test.cpp
CPackConfig.cmake
@@ -161,8 +162,10 @@ tests/queries/0_stateless/test_*
tests/queries/0_stateless/*.binary
tests/queries/0_stateless/*.generated-expect
tests/queries/0_stateless/*.expect.history
+tests/integration/**/_gen
# rust
/rust/**/target
# It is autogenerated from *.in
/rust/**/.cargo/config.toml
+/rust/**/vendor
diff --git a/.gitmodules b/.gitmodules
index 151dc28c55b..30085fb8dd4 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -258,9 +258,6 @@
[submodule "contrib/wyhash"]
path = contrib/wyhash
url = https://github.com/wangyi-fudan/wyhash
-[submodule "contrib/hashidsxx"]
- path = contrib/hashidsxx
- url = https://github.com/schoentoon/hashidsxx
[submodule "contrib/nats-io"]
path = contrib/nats-io
url = https://github.com/ClickHouse/nats.c
@@ -343,3 +340,6 @@
[submodule "contrib/c-ares"]
path = contrib/c-ares
url = https://github.com/c-ares/c-ares.git
+[submodule "contrib/incbin"]
+ path = contrib/incbin
+ url = https://github.com/graphitemaster/incbin.git
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 72372c8fac4..bf6b309ef2c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
### Table of Contents
+**[ClickHouse release v23.6, 2023-06-30](#236)**
**[ClickHouse release v23.5, 2023-06-08](#235)**
**[ClickHouse release v23.4, 2023-04-26](#234)**
**[ClickHouse release v23.3 LTS, 2023-03-30](#233)**
@@ -8,6 +9,106 @@
# 2023 Changelog
+### ClickHouse release 23.6, 2023-06-29
+
+#### Backward Incompatible Change
+* Delete feature `do_not_evict_index_and_mark_files` in the fs cache. This feature was only making things worse. [#51253](https://github.com/ClickHouse/ClickHouse/pull/51253) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Remove ALTER support for experimental LIVE VIEW. [#51287](https://github.com/ClickHouse/ClickHouse/pull/51287) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Decrease the default values for `http_max_field_value_size` and `http_max_field_name_size` to 128 KiB. [#51163](https://github.com/ClickHouse/ClickHouse/pull/51163) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* CGroups metrics related to CPU are replaced with one metric, `CGroupMaxCPU` for better usability. The `Normalized` CPU usage metrics will be normalized to CGroups limits instead of the total number of CPUs when they are set. This closes [#50836](https://github.com/ClickHouse/ClickHouse/issues/50836). [#50835](https://github.com/ClickHouse/ClickHouse/pull/50835) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### New Feature
+* The function `transform` as well as `CASE` with value matching started to support all data types. This closes [#29730](https://github.com/ClickHouse/ClickHouse/issues/29730). This closes [#32387](https://github.com/ClickHouse/ClickHouse/issues/32387). This closes [#50827](https://github.com/ClickHouse/ClickHouse/issues/50827). This closes [#31336](https://github.com/ClickHouse/ClickHouse/issues/31336). This closes [#40493](https://github.com/ClickHouse/ClickHouse/issues/40493). [#51351](https://github.com/ClickHouse/ClickHouse/pull/51351) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Added option `--rename_files_after_processing `. This closes [#34207](https://github.com/ClickHouse/ClickHouse/issues/34207). [#49626](https://github.com/ClickHouse/ClickHouse/pull/49626) ([alekseygolub](https://github.com/alekseygolub)).
+* Add support for `TRUNCATE` modifier in `INTO OUTFILE` clause. Suggest using `APPEND` or `TRUNCATE` for `INTO OUTFILE` when file exists. [#50950](https://github.com/ClickHouse/ClickHouse/pull/50950) ([alekar](https://github.com/alekar)).
+* Add table engine `Redis` and table function `redis`. It allows querying external Redis servers. [#50150](https://github.com/ClickHouse/ClickHouse/pull/50150) ([JackyWoo](https://github.com/JackyWoo)).
+* Allow to skip empty files in file/s3/url/hdfs table functions using settings `s3_skip_empty_files`, `hdfs_skip_empty_files`, `engine_file_skip_empty_files`, `engine_url_skip_empty_files`. [#50364](https://github.com/ClickHouse/ClickHouse/pull/50364) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add a new setting named `use_mysql_types_in_show_columns` to alter the `SHOW COLUMNS` SQL statement to display MySQL equivalent types when a client is connected via the MySQL compatibility port. [#49577](https://github.com/ClickHouse/ClickHouse/pull/49577) ([Thomas Panetti](https://github.com/tpanetti)).
+* Clickhouse-client can now be called with a connection string instead of "--host", "--port", "--user" etc. [#50689](https://github.com/ClickHouse/ClickHouse/pull/50689) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
+* Add setting `session_timezone`; it is used as the default timezone for a session when not explicitly specified. [#44149](https://github.com/ClickHouse/ClickHouse/pull/44149) ([Andrey Zvonov](https://github.com/zvonand)).
+* Codec DEFLATE_QPL is now controlled via server setting "enable_deflate_qpl_codec" (default: false) instead of setting "allow_experimental_codecs". This marks DEFLATE_QPL non-experimental. [#50775](https://github.com/ClickHouse/ClickHouse/pull/50775) ([Robert Schulze](https://github.com/rschu1ze)).
+
+#### Performance Improvement
+* Improved scheduling of merge selecting and cleanup tasks in `ReplicatedMergeTree`. The tasks will not be executed too frequently when there's nothing to merge or cleanup. Added settings `max_merge_selecting_sleep_ms`, `merge_selecting_sleep_slowdown_factor`, `max_cleanup_delay_period` and `cleanup_thread_preferred_points_per_iteration`. It should close [#31919](https://github.com/ClickHouse/ClickHouse/issues/31919). [#50107](https://github.com/ClickHouse/ClickHouse/pull/50107) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Make filter push down through cross join. [#50605](https://github.com/ClickHouse/ClickHouse/pull/50605) ([Han Fei](https://github.com/hanfei1991)).
+* Improve performance with enabled QueryProfiler using thread-local timer_id instead of global object. [#48778](https://github.com/ClickHouse/ClickHouse/pull/48778) ([Jiebin Sun](https://github.com/jiebinn)).
+* Rewrite CapnProto input/output format to improve its performance. Map column names and CapnProto fields case insensitive, fix reading/writing of nested structure fields. [#49752](https://github.com/ClickHouse/ClickHouse/pull/49752) ([Kruglov Pavel](https://github.com/Avogar)).
+* Optimize parquet write performance for parallel threads. [#50102](https://github.com/ClickHouse/ClickHouse/pull/50102) ([Hongbin Ma](https://github.com/binmahone)).
+* Disable `parallelize_output_from_storages` for processing MATERIALIZED VIEWs and storages with one block only. [#50214](https://github.com/ClickHouse/ClickHouse/pull/50214) ([Azat Khuzhin](https://github.com/azat)).
+* Merge PR [#46558](https://github.com/ClickHouse/ClickHouse/pull/46558). Avoid block permutation during sort if the block is already sorted. [#50697](https://github.com/ClickHouse/ClickHouse/pull/50697) ([Alexey Milovidov](https://github.com/alexey-milovidov), [Maksim Kita](https://github.com/kitaisreal)).
+* Make multiple list requests to ZooKeeper in parallel to speed up reading from system.zookeeper table. [#51042](https://github.com/ClickHouse/ClickHouse/pull/51042) ([Alexander Gololobov](https://github.com/davenger)).
+* Speedup initialization of DateTime lookup tables for time zones. This should reduce startup/connect time of clickhouse-client especially in debug build as it is rather heavy. [#51347](https://github.com/ClickHouse/ClickHouse/pull/51347) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix data lakes slowness because of synchronous head requests. (Related to Iceberg/Deltalake/Hudi being slow with a lot of files). [#50976](https://github.com/ClickHouse/ClickHouse/pull/50976) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Do not read all the columns from right GLOBAL JOIN table. [#50721](https://github.com/ClickHouse/ClickHouse/pull/50721) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+
+#### Experimental Feature
+* Support parallel replicas with the analyzer. [#50441](https://github.com/ClickHouse/ClickHouse/pull/50441) ([Raúl Marín](https://github.com/Algunenano)).
+* Add random sleep before large merges/mutations execution to split load more evenly between replicas in case of zero-copy replication. [#51282](https://github.com/ClickHouse/ClickHouse/pull/51282) ([alesapin](https://github.com/alesapin)).
+* Do not replicate `ALTER PARTITION` queries and mutations through `Replicated` database if it has only one shard and the underlying table is `ReplicatedMergeTree`. [#51049](https://github.com/ClickHouse/ClickHouse/pull/51049) ([Alexander Tokmakov](https://github.com/tavplubix)).
+
+#### Improvement
+* Relax the thresholds for "too many parts" to be more modern. Return the backpressure during long-running insert queries. [#50856](https://github.com/ClickHouse/ClickHouse/pull/50856) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Allow to cast IPv6 to IPv4 address for CIDR ::ffff:0:0/96 (IPv4-mapped addresses). [#49759](https://github.com/ClickHouse/ClickHouse/pull/49759) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Update MongoDB protocol to support MongoDB 5.1 version and newer. Support for the versions with the old protocol (<3.6) is preserved. Closes [#45621](https://github.com/ClickHouse/ClickHouse/issues/45621), [#49879](https://github.com/ClickHouse/ClickHouse/issues/49879). [#50061](https://github.com/ClickHouse/ClickHouse/pull/50061) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Add setting `input_format_max_bytes_to_read_for_schema_inference` to limit the number of bytes to read in schema inference. Closes [#50577](https://github.com/ClickHouse/ClickHouse/issues/50577). [#50592](https://github.com/ClickHouse/ClickHouse/pull/50592) ([Kruglov Pavel](https://github.com/Avogar)).
+* Respect setting `input_format_null_as_default` in schema inference. [#50602](https://github.com/ClickHouse/ClickHouse/pull/50602) ([Kruglov Pavel](https://github.com/Avogar)).
+* Allow to skip trailing empty lines in CSV/TSV/CustomSeparated formats via settings `input_format_csv_skip_trailing_empty_lines`, `input_format_tsv_skip_trailing_empty_lines` and `input_format_custom_skip_trailing_empty_lines` (disabled by default). Closes [#49315](https://github.com/ClickHouse/ClickHouse/issues/49315). [#50635](https://github.com/ClickHouse/ClickHouse/pull/50635) ([Kruglov Pavel](https://github.com/Avogar)).
+* Functions "toDateOrDefault|OrNull" and "accuateCast[OrDefault|OrNull]" now correctly parse numeric arguments. [#50709](https://github.com/ClickHouse/ClickHouse/pull/50709) ([Dmitry Kardymon](https://github.com/kardymonds)).
+* Support CSV with whitespace or `\t` field delimiters, and these delimiters are supported in Spark. [#50712](https://github.com/ClickHouse/ClickHouse/pull/50712) ([KevinyhZou](https://github.com/KevinyhZou)).
+* Settings `number_of_mutations_to_delay` and `number_of_mutations_to_throw` are enabled by default now with values 500 and 1000 respectively. [#50726](https://github.com/ClickHouse/ClickHouse/pull/50726) ([Anton Popov](https://github.com/CurtizJ)).
+* The dashboard correctly shows missing values. This closes [#50831](https://github.com/ClickHouse/ClickHouse/issues/50831). [#50832](https://github.com/ClickHouse/ClickHouse/pull/50832) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Added the possibility to use date and time arguments in the syslog timestamp format in functions `parseDateTimeBestEffort*` and `parseDateTime64BestEffort*`. [#50925](https://github.com/ClickHouse/ClickHouse/pull/50925) ([Victor Krasnov](https://github.com/sirvickr)).
+* Command line parameter "--password" in clickhouse-client can now be specified only once. [#50966](https://github.com/ClickHouse/ClickHouse/pull/50966) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
+* Use `hash_of_all_files` from `system.parts` to check identity of parts during on-cluster backups. [#50997](https://github.com/ClickHouse/ClickHouse/pull/50997) ([Vitaly Baranov](https://github.com/vitlibar)).
+* The system table zookeeper_connection connected_time identifies the time when the connection is established (standard format), and session_uptime_elapsed_seconds is added, which labels the duration of the established connection session (in seconds). [#51026](https://github.com/ClickHouse/ClickHouse/pull/51026) ([郭小龙](https://github.com/guoxiaolongzte)).
+* Improve the progress bar for file/s3/hdfs/url table functions by using chunk size from source data and using incremental total size counting in each thread. Fix the progress bar for *Cluster functions. This closes [#47250](https://github.com/ClickHouse/ClickHouse/issues/47250). [#51088](https://github.com/ClickHouse/ClickHouse/pull/51088) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add total_bytes_to_read to the Progress packet in TCP protocol for better Progress bar. [#51158](https://github.com/ClickHouse/ClickHouse/pull/51158) ([Kruglov Pavel](https://github.com/Avogar)).
+* Better checking of data parts on disks with filesystem cache. [#51164](https://github.com/ClickHouse/ClickHouse/pull/51164) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix sometimes not correct current_elements_num in fs cache. [#51242](https://github.com/ClickHouse/ClickHouse/pull/51242) ([Kseniia Sumarokova](https://github.com/kssenii)).
+
+#### Build/Testing/Packaging Improvement
+* Add embedded keeper-client to standalone keeper binary. [#50964](https://github.com/ClickHouse/ClickHouse/pull/50964) ([pufit](https://github.com/pufit)).
+* Actual LZ4 version is used now. [#50621](https://github.com/ClickHouse/ClickHouse/pull/50621) ([Nikita Taranov](https://github.com/nickitat)).
+* ClickHouse server will print the list of changed settings on fatal errors. This closes [#51137](https://github.com/ClickHouse/ClickHouse/issues/51137). [#51138](https://github.com/ClickHouse/ClickHouse/pull/51138) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Allow building ClickHouse with clang-17. [#51300](https://github.com/ClickHouse/ClickHouse/pull/51300) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* [SQLancer](https://github.com/sqlancer/sqlancer) check is considered stable as bugs that were triggered by it are fixed. Now failures of SQLancer check will be reported as failed check status. [#51340](https://github.com/ClickHouse/ClickHouse/pull/51340) ([Ilya Yatsishin](https://github.com/qoega)).
+* Split huge `RUN` in Dockerfile into smaller conditional. Install the necessary tools on demand in the same `RUN` layer, and remove them after that. Upgrade the OS only once at the beginning. Use a modern way to check the signed repository. Downgrade the base repo to ubuntu:20.04 to address the issues on older docker versions. Upgrade golang version to address golang vulnerabilities. [#51504](https://github.com/ClickHouse/ClickHouse/pull/51504) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Report loading status for executable dictionaries correctly [#48775](https://github.com/ClickHouse/ClickHouse/pull/48775) ([Anton Kozlov](https://github.com/tonickkozlov)).
+* Proper mutation of skip indices and projections [#50104](https://github.com/ClickHouse/ClickHouse/pull/50104) ([Amos Bird](https://github.com/amosbird)).
+* Cleanup moving parts [#50489](https://github.com/ClickHouse/ClickHouse/pull/50489) ([vdimir](https://github.com/vdimir)).
+* Fix backward compatibility for IP types hashing in aggregate functions [#50551](https://github.com/ClickHouse/ClickHouse/pull/50551) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix Log family table return wrong rows count after truncate [#50585](https://github.com/ClickHouse/ClickHouse/pull/50585) ([flynn](https://github.com/ucasfl)).
+* Fix bug in `uniqExact` parallel merging [#50590](https://github.com/ClickHouse/ClickHouse/pull/50590) ([Nikita Taranov](https://github.com/nickitat)).
+* Revert recent grace hash join changes [#50699](https://github.com/ClickHouse/ClickHouse/pull/50699) ([vdimir](https://github.com/vdimir)).
+* Query Cache: Try to fix bad cast from `ColumnConst` to `ColumnVector` [#50704](https://github.com/ClickHouse/ClickHouse/pull/50704) ([Robert Schulze](https://github.com/rschu1ze)).
+* Avoid storing logs in Keeper containing unknown operation [#50751](https://github.com/ClickHouse/ClickHouse/pull/50751) ([Antonio Andelic](https://github.com/antonio2368)).
+* SummingMergeTree support for DateTime64 [#50797](https://github.com/ClickHouse/ClickHouse/pull/50797) ([Jordi Villar](https://github.com/jrdi)).
+* Add compatibility setting for non-const timezones [#50834](https://github.com/ClickHouse/ClickHouse/pull/50834) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix hashing of LDAP params in the cache entries [#50865](https://github.com/ClickHouse/ClickHouse/pull/50865) ([Julian Maicher](https://github.com/jmaicher)).
+* Fallback to parsing big integer from String instead of exception in Parquet format [#50873](https://github.com/ClickHouse/ClickHouse/pull/50873) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix checking the lock file too often while writing a backup [#50889](https://github.com/ClickHouse/ClickHouse/pull/50889) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Do not apply projection if read-in-order was enabled. [#50923](https://github.com/ClickHouse/ClickHouse/pull/50923) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix race in the Azure blob storage iterator [#50936](https://github.com/ClickHouse/ClickHouse/pull/50936) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix erroneous `sort_description` propagation in `CreatingSets` [#50955](https://github.com/ClickHouse/ClickHouse/pull/50955) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix Iceberg v2 optional metadata parsing [#50974](https://github.com/ClickHouse/ClickHouse/pull/50974) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* MaterializedMySQL: Keep parentheses for empty table overrides [#50977](https://github.com/ClickHouse/ClickHouse/pull/50977) ([Val Doroshchuk](https://github.com/valbok)).
+* Fix crash in BackupCoordinationStageSync::setError() [#51012](https://github.com/ClickHouse/ClickHouse/pull/51012) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix subtly broken copy-on-write of ColumnLowCardinality dictionary [#51064](https://github.com/ClickHouse/ClickHouse/pull/51064) ([Michael Kolupaev](https://github.com/al13n321)).
+* Generate safe IVs [#51086](https://github.com/ClickHouse/ClickHouse/pull/51086) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Fix ineffective query cache for SELECTs with subqueries [#51132](https://github.com/ClickHouse/ClickHouse/pull/51132) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix Set index with constant nullable comparison. [#51205](https://github.com/ClickHouse/ClickHouse/pull/51205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix a crash in s3 and s3Cluster functions [#51209](https://github.com/ClickHouse/ClickHouse/pull/51209) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix a crash with compiled expressions [#51231](https://github.com/ClickHouse/ClickHouse/pull/51231) ([LiuNeng](https://github.com/liuneng1994)).
+* Fix use-after-free in StorageURL when switching URLs [#51260](https://github.com/ClickHouse/ClickHouse/pull/51260) ([Michael Kolupaev](https://github.com/al13n321)).
+* Updated check for parameterized view [#51272](https://github.com/ClickHouse/ClickHouse/pull/51272) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix multiple writing of same file to backup [#51299](https://github.com/ClickHouse/ClickHouse/pull/51299) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix fuzzer failure in ActionsDAG [#51301](https://github.com/ClickHouse/ClickHouse/pull/51301) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Remove garbage from function `transform` [#51350](https://github.com/ClickHouse/ClickHouse/pull/51350) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+
### ClickHouse release 23.5, 2023-06-08
#### Upgrade Notes
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5d6ed75bb29..45c3c422d7a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -87,7 +87,6 @@ if (ENABLE_FUZZING)
set (ENABLE_CLICKHOUSE_ODBC_BRIDGE OFF)
set (ENABLE_LIBRARIES 0)
set (ENABLE_SSL 1)
- set (USE_UNWIND ON)
set (ENABLE_EMBEDDED_COMPILER 0)
set (ENABLE_EXAMPLES 0)
set (ENABLE_UTILS 0)
@@ -344,9 +343,9 @@ if (COMPILER_CLANG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-absolute-paths")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths")
- if (NOT ENABLE_TESTS AND NOT SANITIZE)
+ if (NOT ENABLE_TESTS AND NOT SANITIZE AND OS_LINUX)
# https://clang.llvm.org/docs/ThinLTO.html
- # Applies to clang only.
+ # Applies to clang and linux only.
# Disabled when building with tests or sanitizers.
option(ENABLE_THINLTO "Clang-specific link time optimization" ON)
endif()
diff --git a/README.md b/README.md
index 9561458ba37..f0a7dbe2408 100644
--- a/README.md
+++ b/README.md
@@ -16,28 +16,31 @@ curl https://clickhouse.com/ | sh
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
* [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
* [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events.
-* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation.
-* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev.
+* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlighting and navigation.
+* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev.
+* [Static Analysis (SonarCloud)](https://sonarcloud.io/project/issues?resolved=false&id=ClickHouse_ClickHouse) proposes C++ quality improvements.
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
## Upcoming Events
-* [**v23.5 Release Webinar**](https://clickhouse.com/company/events/v23-5-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-05) - Jun 8 - 23.5 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
-* [**ClickHouse Meetup in Bangalore**](https://www.meetup.com/clickhouse-bangalore-user-group/events/293740066/) - Jun 7
-* [**ClickHouse Meetup in San Francisco**](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/293426725/) - Jun 7
+* [**v23.7 Release Webinar**](https://clickhouse.com/company/events/v23-7-community-release-call?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-07) - Jul 27 - 23.7 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
+* [**ClickHouse Meetup in Boston**](https://www.meetup.com/clickhouse-boston-user-group/events/293913596) - Jul 18
+* [**ClickHouse Meetup in NYC**](https://www.meetup.com/clickhouse-new-york-user-group/events/293913441) - Jul 19
+* [**ClickHouse Meetup in Toronto**](https://www.meetup.com/clickhouse-toronto-user-group/events/294183127) - Jul 20
+* [**ClickHouse Meetup in Singapore**](https://www.meetup.com/clickhouse-singapore-meetup-group/events/294428050/) - Jul 27
+* [**ClickHouse Meetup in Paris**](https://www.meetup.com/clickhouse-france-user-group/events/294283460) - Sep 12
-
-Also, keep an eye out for upcoming meetups in Amsterdam, Boston, NYC, Beijing, and Toronto. Somewhere else you want us to be? Please feel free to reach out to tyler clickhouse com.
+Also, keep an eye out for upcoming meetups around the world. Somewhere else you want us to be? Please feel free to reach out to tyler clickhouse com.
## Recent Recordings
* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
-* **Recording available**: [**v23.4 Release Webinar**](https://www.youtube.com/watch?v=4rrf6bk_mOg) Faster Parquet Reading, Asynchonous Connections to Reoplicas, Trailing Comma before FROM, extractKeyValuePairs, integrations updates, and so much more! Watch it now!
+* **Recording available**: [**v23.6 Release Webinar**](https://www.youtube.com/watch?v=cuf_hYn7dqU) All the features of 23.6, one convenient video! Watch it now!
* **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)
- ## Interested in joining ClickHouse and making it your full time job?
+ ## Interested in joining ClickHouse and making it your full-time job?
-We are a globally diverse and distributed team, united behind a common goal of creating industry-leading, real-time analytics. Here, you will have an opportunity to solve some of the most cutting edge technical challenges and have direct ownership of your work and vision. If you are a contributor by nature, a thinker as well as a doer - we’ll definitely click!
+We are a globally diverse and distributed team, united behind a common goal of creating industry-leading, real-time analytics. Here, you will have an opportunity to solve some of the most cutting-edge technical challenges and have direct ownership of your work and vision. If you are a contributor by nature, a thinker and a doer - we’ll definitely click!
Check out our **current openings** here: https://clickhouse.com/company/careers
diff --git a/SECURITY.md b/SECURITY.md
index 1864eb6e9e5..4ba5f13d09c 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -13,6 +13,7 @@ The following versions of ClickHouse server are currently being supported with s
| Version | Supported |
|:-|:-|
+| 23.6 | ✔️ |
| 23.5 | ✔️ |
| 23.4 | ✔️ |
| 23.3 | ✔️ |
diff --git a/base/base/IPv4andIPv6.h b/base/base/IPv4andIPv6.h
index 7b745ec7b84..e2f93b54124 100644
--- a/base/base/IPv4andIPv6.h
+++ b/base/base/IPv4andIPv6.h
@@ -2,21 +2,23 @@
#include
#include
+#include
#include
namespace DB
{
- using IPv4 = StrongTypedef;
+ struct IPv4 : StrongTypedef
+ {
+ using StrongTypedef::StrongTypedef;
+ using StrongTypedef::operator=;
+ constexpr explicit IPv4(UInt64 value): StrongTypedef(static_cast(value)) {}
+ };
struct IPv6 : StrongTypedef
{
- constexpr IPv6() = default;
- constexpr explicit IPv6(const UInt128 & x) : StrongTypedef(x) {}
- constexpr explicit IPv6(UInt128 && x) : StrongTypedef(std::move(x)) {}
-
- IPv6 & operator=(const UInt128 & rhs) { StrongTypedef::operator=(rhs); return *this; }
- IPv6 & operator=(UInt128 && rhs) { StrongTypedef::operator=(std::move(rhs)); return *this; }
+ using StrongTypedef::StrongTypedef;
+ using StrongTypedef::operator=;
bool operator<(const IPv6 & rhs) const
{
@@ -54,12 +56,22 @@ namespace DB
namespace std
{
+ /// For historical reasons we hash IPv6 as a FixedString(16)
template <>
struct hash
{
size_t operator()(const DB::IPv6 & x) const
{
- return std::hash()(x.toUnderType());
+ return std::hash{}(std::string_view(reinterpret_cast(&x.toUnderType()), IPV6_BINARY_LENGTH));
+ }
+ };
+
+ template <>
+ struct hash
+ {
+ size_t operator()(const DB::IPv4 & x) const
+ {
+ return std::hash()(x.toUnderType());
}
};
}
diff --git a/base/base/bit_cast.h b/base/base/bit_cast.h
index 5373ead36e8..4783a84586b 100644
--- a/base/base/bit_cast.h
+++ b/base/base/bit_cast.h
@@ -7,7 +7,13 @@
/** Returns value `from` converted to type `To` while retaining bit representation.
* `To` and `From` must satisfy `CopyConstructible`.
+ *
* In contrast to std::bit_cast can cast types of different width.
+ *
+ * Note: for signed types of narrower size, the casted result is zero-extended
+ * instead of sign-extended as with regular static_cast.
+ * For example, -1 Int8 (represented as 0xFF) bit_casted to UInt64
+ * gives 255 (represented as 0x00000000000000FF) instead of 0xFFFFFFFFFFFFFFFF
*/
template
std::decay_t bit_cast(const From & from)
diff --git a/base/base/find_symbols.h b/base/base/find_symbols.h
index a8747ecc9b7..fda94edaa88 100644
--- a/base/base/find_symbols.h
+++ b/base/base/find_symbols.h
@@ -2,6 +2,7 @@
#include
#include
+#include
#if defined(__SSE2__)
#include
@@ -447,7 +448,7 @@ inline char * find_last_not_symbols_or_null(char * begin, char * end)
/// See https://github.com/boostorg/algorithm/issues/63
/// And https://bugs.llvm.org/show_bug.cgi?id=41141
template
-inline void splitInto(To & to, const std::string & what, bool token_compress = false)
+inline To & splitInto(To & to, std::string_view what, bool token_compress = false)
{
const char * pos = what.data();
const char * end = pos + what.size();
@@ -463,4 +464,6 @@ inline void splitInto(To & to, const std::string & what, bool token_compress = f
else
pos = delimiter_or_end;
}
+
+ return to;
}
diff --git a/base/base/getThreadId.cpp b/base/base/getThreadId.cpp
index b6c22bb8856..a42d79c5698 100644
--- a/base/base/getThreadId.cpp
+++ b/base/base/getThreadId.cpp
@@ -15,25 +15,34 @@
static thread_local uint64_t current_tid = 0;
+
+static void setCurrentThreadId()
+{
+#if defined(OS_ANDROID)
+ current_tid = gettid();
+#elif defined(OS_LINUX)
+ current_tid = static_cast(syscall(SYS_gettid)); /// This call is always successful. - man gettid
+#elif defined(OS_FREEBSD)
+ current_tid = pthread_getthreadid_np();
+#elif defined(OS_SUNOS)
+ // On Solaris-derived systems, this returns the ID of the LWP, analogous
+ // to a thread.
+ current_tid = static_cast(pthread_self());
+#else
+ if (0 != pthread_threadid_np(nullptr, ¤t_tid))
+ throw std::logic_error("pthread_threadid_np returned error");
+#endif
+}
+
uint64_t getThreadId()
{
if (!current_tid)
- {
-#if defined(OS_ANDROID)
- current_tid = gettid();
-#elif defined(OS_LINUX)
- current_tid = static_cast(syscall(SYS_gettid)); /// This call is always successful. - man gettid
-#elif defined(OS_FREEBSD)
- current_tid = pthread_getthreadid_np();
-#elif defined(OS_SUNOS)
- // On Solaris-derived systems, this returns the ID of the LWP, analogous
- // to a thread.
- current_tid = static_cast(pthread_self());
-#else
- if (0 != pthread_threadid_np(nullptr, ¤t_tid))
- throw std::logic_error("pthread_threadid_np returned error");
-#endif
- }
+ setCurrentThreadId();
return current_tid;
}
+
+void updateCurrentThreadIdAfterFork()
+{
+ setCurrentThreadId();
+}
diff --git a/base/base/getThreadId.h b/base/base/getThreadId.h
index a1b5ff5f3e8..f90c76029e1 100644
--- a/base/base/getThreadId.h
+++ b/base/base/getThreadId.h
@@ -3,3 +3,5 @@
/// Obtain thread id from OS. The value is cached in thread local variable.
uint64_t getThreadId();
+
+void updateCurrentThreadIdAfterFork();
diff --git a/base/base/hex.h b/base/base/hex.h
index b8cf95db893..937218fec5a 100644
--- a/base/base/hex.h
+++ b/base/base/hex.h
@@ -4,212 +4,288 @@
#include
#include "types.h"
-/// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly.
+namespace CityHash_v1_0_2 { struct uint128; }
-constexpr inline std::string_view hex_digit_to_char_uppercase_table = "0123456789ABCDEF";
-constexpr inline std::string_view hex_digit_to_char_lowercase_table = "0123456789abcdef";
+namespace wide
+{
+ template
+ class integer;
+}
+
+namespace impl
+{
+ /// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly.
+ constexpr inline std::string_view hex_digit_to_char_uppercase_table = "0123456789ABCDEF";
+ constexpr inline std::string_view hex_digit_to_char_lowercase_table = "0123456789abcdef";
+
+ /// Maps 0..255 to 00..FF or 00..ff correspondingly.
+ constexpr inline std::string_view hex_byte_to_char_uppercase_table = //
+ "000102030405060708090A0B0C0D0E0F"
+ "101112131415161718191A1B1C1D1E1F"
+ "202122232425262728292A2B2C2D2E2F"
+ "303132333435363738393A3B3C3D3E3F"
+ "404142434445464748494A4B4C4D4E4F"
+ "505152535455565758595A5B5C5D5E5F"
+ "606162636465666768696A6B6C6D6E6F"
+ "707172737475767778797A7B7C7D7E7F"
+ "808182838485868788898A8B8C8D8E8F"
+ "909192939495969798999A9B9C9D9E9F"
+ "A0A1A2A3A4A5A6A7A8A9AAABACADAEAF"
+ "B0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF"
+ "C0C1C2C3C4C5C6C7C8C9CACBCCCDCECF"
+ "D0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF"
+ "E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEF"
+ "F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF";
+
+ constexpr inline std::string_view hex_byte_to_char_lowercase_table = //
+ "000102030405060708090a0b0c0d0e0f"
+ "101112131415161718191a1b1c1d1e1f"
+ "202122232425262728292a2b2c2d2e2f"
+ "303132333435363738393a3b3c3d3e3f"
+ "404142434445464748494a4b4c4d4e4f"
+ "505152535455565758595a5b5c5d5e5f"
+ "606162636465666768696a6b6c6d6e6f"
+ "707172737475767778797a7b7c7d7e7f"
+ "808182838485868788898a8b8c8d8e8f"
+ "909192939495969798999a9b9c9d9e9f"
+ "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
+ "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"
+ "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf"
+ "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf"
+ "e0e1e2e3e4e5e6e7e8e9eaebecedeeef"
+ "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff";
+
+ /// Maps 0..255 to 00000000..11111111 correspondingly.
+ constexpr inline std::string_view bin_byte_to_char_table = //
+ "0000000000000001000000100000001100000100000001010000011000000111"
+ "0000100000001001000010100000101100001100000011010000111000001111"
+ "0001000000010001000100100001001100010100000101010001011000010111"
+ "0001100000011001000110100001101100011100000111010001111000011111"
+ "0010000000100001001000100010001100100100001001010010011000100111"
+ "0010100000101001001010100010101100101100001011010010111000101111"
+ "0011000000110001001100100011001100110100001101010011011000110111"
+ "0011100000111001001110100011101100111100001111010011111000111111"
+ "0100000001000001010000100100001101000100010001010100011001000111"
+ "0100100001001001010010100100101101001100010011010100111001001111"
+ "0101000001010001010100100101001101010100010101010101011001010111"
+ "0101100001011001010110100101101101011100010111010101111001011111"
+ "0110000001100001011000100110001101100100011001010110011001100111"
+ "0110100001101001011010100110101101101100011011010110111001101111"
+ "0111000001110001011100100111001101110100011101010111011001110111"
+ "0111100001111001011110100111101101111100011111010111111001111111"
+ "1000000010000001100000101000001110000100100001011000011010000111"
+ "1000100010001001100010101000101110001100100011011000111010001111"
+ "1001000010010001100100101001001110010100100101011001011010010111"
+ "1001100010011001100110101001101110011100100111011001111010011111"
+ "1010000010100001101000101010001110100100101001011010011010100111"
+ "1010100010101001101010101010101110101100101011011010111010101111"
+ "1011000010110001101100101011001110110100101101011011011010110111"
+ "1011100010111001101110101011101110111100101111011011111010111111"
+ "1100000011000001110000101100001111000100110001011100011011000111"
+ "1100100011001001110010101100101111001100110011011100111011001111"
+ "1101000011010001110100101101001111010100110101011101011011010111"
+ "1101100011011001110110101101101111011100110111011101111011011111"
+ "1110000011100001111000101110001111100100111001011110011011100111"
+ "1110100011101001111010101110101111101100111011011110111011101111"
+ "1111000011110001111100101111001111110100111101011111011011110111"
+ "1111100011111001111110101111101111111100111111011111111011111111";
+
+ /// Maps 0..9, A..F, a..f to 0..15. Other chars are mapped to implementation specific value.
+ constexpr inline std::string_view hex_char_to_digit_table
+ = {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" //0-9
+ "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //A-Z
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //a-z
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff",
+ 256};
+
+ /// Converts a hex digit '0'..'f' or '0'..'F' to its value 0..15.
+ constexpr UInt8 unhexDigit(char c)
+ {
+ return hex_char_to_digit_table[static_cast(c)];
+ }
+
+ /// Converts an unsigned integer in the native endian to hexadecimal representation and back. Used as a base class for HexConversion.
+ template
+ struct HexConversionUInt
+ {
+ static const constexpr size_t num_hex_digits = sizeof(TUInt) * 2;
+
+ static void hex(TUInt uint_, char * out, std::string_view table)
+ {
+ union
+ {
+ TUInt value;
+ UInt8 uint8[sizeof(TUInt)];
+ };
+
+ value = uint_;
+
+ for (size_t i = 0; i < sizeof(TUInt); ++i)
+ {
+ if constexpr (std::endian::native == std::endian::little)
+ memcpy(out + i * 2, &table[static_cast(uint8[sizeof(TUInt) - 1 - i]) * 2], 2);
+ else
+ memcpy(out + i * 2, &table[static_cast(uint8[i]) * 2], 2);
+ }
+ }
+
+ static TUInt unhex(const char * data)
+ {
+ TUInt res;
+ if constexpr (sizeof(TUInt) == 1)
+ {
+ res = static_cast(unhexDigit(data[0])) * 0x10 + static_cast(unhexDigit(data[1]));
+ }
+ else if constexpr (sizeof(TUInt) == 2)
+ {
+ res = static_cast(unhexDigit(data[0])) * 0x1000 + static_cast(unhexDigit(data[1])) * 0x100
+ + static_cast(unhexDigit(data[2])) * 0x10 + static_cast(unhexDigit(data[3]));
+ }
+ else if constexpr ((sizeof(TUInt) <= 8) || ((sizeof(TUInt) % 8) != 0))
+ {
+ res = 0;
+ for (size_t i = 0; i < sizeof(TUInt) * 2; ++i, ++data)
+ {
+ res <<= 4;
+ res += unhexDigit(*data);
+ }
+ }
+ else
+ {
+ res = 0;
+ for (size_t i = 0; i < sizeof(TUInt) / 8; ++i, data += 16)
+ {
+ res <<= 64;
+ res += HexConversionUInt::unhex(data);
+ }
+ }
+ return res;
+ }
+ };
+
+ /// Helper template class to convert a value of any supported type to hexadecimal representation and back.
+ template
+ struct HexConversion;
+
+ template
+ struct HexConversion>> : public HexConversionUInt {};
+
+ template
+ struct HexConversion> : public HexConversionUInt> {};
+
+ template /// Partial specialization here allows not to include in this header.
+ struct HexConversion>>
+ {
+ static const constexpr size_t num_hex_digits = 32;
+
+ static void hex(const CityHashUInt128 & uint_, char * out, std::string_view table)
+ {
+ HexConversion::hex(uint_.high64, out, table);
+ HexConversion::hex(uint_.low64, out + 16, table);
+ }
+
+ static CityHashUInt128 unhex(const char * data)
+ {
+ CityHashUInt128 res;
+ res.high64 = HexConversion::unhex(data);
+ res.low64 = HexConversion::unhex(data + 16);
+ return res;
+ }
+ };
+}
+
+/// Produces a hexadecimal representation of an integer value with leading zeros (for checksums).
+/// The function supports native integer types, wide::integer, CityHash_v1_0_2::uint128.
+/// It can be used with signed types as well, however they are written as corresponding unsigned numbers
+/// using two's complement (i.e. for example "-1" is written as "0xFF", not as "-0x01").
+template
+void writeHexUIntUppercase(const T & value, char * out)
+{
+ impl::HexConversion::hex(value, out, impl::hex_byte_to_char_uppercase_table);
+}
+
+template
+void writeHexUIntLowercase(const T & value, char * out)
+{
+ impl::HexConversion::hex(value, out, impl::hex_byte_to_char_lowercase_table);
+}
+
+template
+std::string getHexUIntUppercase(const T & value)
+{
+ std::string res(impl::HexConversion::num_hex_digits, '\0');
+ writeHexUIntUppercase(value, res.data());
+ return res;
+}
+
+template
+std::string getHexUIntLowercase(const T & value)
+{
+ std::string res(impl::HexConversion::num_hex_digits, '\0');
+ writeHexUIntLowercase(value, res.data());
+ return res;
+}
constexpr char hexDigitUppercase(unsigned char c)
{
- return hex_digit_to_char_uppercase_table[c];
+ return impl::hex_digit_to_char_uppercase_table[c];
}
+
constexpr char hexDigitLowercase(unsigned char c)
{
- return hex_digit_to_char_lowercase_table[c];
+ return impl::hex_digit_to_char_lowercase_table[c];
}
-/// Maps 0..255 to 00..FF or 00..ff correspondingly
-
-constexpr inline std::string_view hex_byte_to_char_uppercase_table = //
- "000102030405060708090A0B0C0D0E0F"
- "101112131415161718191A1B1C1D1E1F"
- "202122232425262728292A2B2C2D2E2F"
- "303132333435363738393A3B3C3D3E3F"
- "404142434445464748494A4B4C4D4E4F"
- "505152535455565758595A5B5C5D5E5F"
- "606162636465666768696A6B6C6D6E6F"
- "707172737475767778797A7B7C7D7E7F"
- "808182838485868788898A8B8C8D8E8F"
- "909192939495969798999A9B9C9D9E9F"
- "A0A1A2A3A4A5A6A7A8A9AAABACADAEAF"
- "B0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF"
- "C0C1C2C3C4C5C6C7C8C9CACBCCCDCECF"
- "D0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF"
- "E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEF"
- "F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF";
-
-constexpr inline std::string_view hex_byte_to_char_lowercase_table = //
- "000102030405060708090a0b0c0d0e0f"
- "101112131415161718191a1b1c1d1e1f"
- "202122232425262728292a2b2c2d2e2f"
- "303132333435363738393a3b3c3d3e3f"
- "404142434445464748494a4b4c4d4e4f"
- "505152535455565758595a5b5c5d5e5f"
- "606162636465666768696a6b6c6d6e6f"
- "707172737475767778797a7b7c7d7e7f"
- "808182838485868788898a8b8c8d8e8f"
- "909192939495969798999a9b9c9d9e9f"
- "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
- "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"
- "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf"
- "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf"
- "e0e1e2e3e4e5e6e7e8e9eaebecedeeef"
- "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff";
-
inline void writeHexByteUppercase(UInt8 byte, void * out)
{
- memcpy(out, &hex_byte_to_char_uppercase_table[static_cast(byte) * 2], 2);
+ memcpy(out, &impl::hex_byte_to_char_uppercase_table[static_cast(byte) * 2], 2);
}
inline void writeHexByteLowercase(UInt8 byte, void * out)
{
- memcpy(out, &hex_byte_to_char_lowercase_table[static_cast(byte) * 2], 2);
+ memcpy(out, &impl::hex_byte_to_char_lowercase_table[static_cast(byte) * 2], 2);
}
-constexpr inline std::string_view bin_byte_to_char_table = //
- "0000000000000001000000100000001100000100000001010000011000000111"
- "0000100000001001000010100000101100001100000011010000111000001111"
- "0001000000010001000100100001001100010100000101010001011000010111"
- "0001100000011001000110100001101100011100000111010001111000011111"
- "0010000000100001001000100010001100100100001001010010011000100111"
- "0010100000101001001010100010101100101100001011010010111000101111"
- "0011000000110001001100100011001100110100001101010011011000110111"
- "0011100000111001001110100011101100111100001111010011111000111111"
- "0100000001000001010000100100001101000100010001010100011001000111"
- "0100100001001001010010100100101101001100010011010100111001001111"
- "0101000001010001010100100101001101010100010101010101011001010111"
- "0101100001011001010110100101101101011100010111010101111001011111"
- "0110000001100001011000100110001101100100011001010110011001100111"
- "0110100001101001011010100110101101101100011011010110111001101111"
- "0111000001110001011100100111001101110100011101010111011001110111"
- "0111100001111001011110100111101101111100011111010111111001111111"
- "1000000010000001100000101000001110000100100001011000011010000111"
- "1000100010001001100010101000101110001100100011011000111010001111"
- "1001000010010001100100101001001110010100100101011001011010010111"
- "1001100010011001100110101001101110011100100111011001111010011111"
- "1010000010100001101000101010001110100100101001011010011010100111"
- "1010100010101001101010101010101110101100101011011010111010101111"
- "1011000010110001101100101011001110110100101101011011011010110111"
- "1011100010111001101110101011101110111100101111011011111010111111"
- "1100000011000001110000101100001111000100110001011100011011000111"
- "1100100011001001110010101100101111001100110011011100111011001111"
- "1101000011010001110100101101001111010100110101011101011011010111"
- "1101100011011001110110101101101111011100110111011101111011011111"
- "1110000011100001111000101110001111100100111001011110011011100111"
- "1110100011101001111010101110101111101100111011011110111011101111"
- "1111000011110001111100101111001111110100111101011111011011110111"
- "1111100011111001111110101111101111111100111111011111111011111111";
-
-inline void writeBinByte(UInt8 byte, void * out)
+/// Converts a hex representation with leading zeros back to an integer value.
+/// The function supports native integer types, wide::integer, CityHash_v1_0_2::uint128.
+template
+constexpr T unhexUInt(const char * data)
{
- memcpy(out, &bin_byte_to_char_table[static_cast(byte) * 8], 8);
+ return impl::HexConversion::unhex(data);
}
-/// Produces hex representation of an unsigned int with leading zeros (for checksums)
-template
-inline void writeHexUIntImpl(TUInt uint_, char * out, std::string_view table)
-{
- union
- {
- TUInt value;
- UInt8 uint8[sizeof(TUInt)];
- };
-
- value = uint_;
-
- for (size_t i = 0; i < sizeof(TUInt); ++i)
- {
- if constexpr (std::endian::native == std::endian::little)
- memcpy(out + i * 2, &table[static_cast(uint8[sizeof(TUInt) - 1 - i]) * 2], 2);
- else
- memcpy(out + i * 2, &table[static_cast(uint8[i]) * 2], 2);
- }
-}
-
-template
-inline void writeHexUIntUppercase(TUInt uint_, char * out)
-{
- writeHexUIntImpl(uint_, out, hex_byte_to_char_uppercase_table);
-}
-
-template
-inline void writeHexUIntLowercase(TUInt uint_, char * out)
-{
- writeHexUIntImpl(uint_, out, hex_byte_to_char_lowercase_table);
-}
-
-template
-std::string getHexUIntUppercase(TUInt uint_)
-{
- std::string res(sizeof(TUInt) * 2, '\0');
- writeHexUIntUppercase(uint_, res.data());
- return res;
-}
-
-template
-std::string getHexUIntLowercase(TUInt uint_)
-{
- std::string res(sizeof(TUInt) * 2, '\0');
- writeHexUIntLowercase(uint_, res.data());
- return res;
-}
-
-/// Maps 0..9, A..F, a..f to 0..15. Other chars are mapped to implementation specific value.
-
-constexpr inline std::string_view hex_char_to_digit_table
- = {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" //0-9
- "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //A-Z
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //a-z
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
- "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff",
- 256};
-
+/// Converts a hexadecimal digit '0'..'f' or '0'..'F' to UInt8.
constexpr UInt8 unhex(char c)
{
- return hex_char_to_digit_table[static_cast(c)];
+ return impl::unhexDigit(c);
}
+/// Converts two hexadecimal digits to UInt8.
constexpr UInt8 unhex2(const char * data)
{
- return static_cast(unhex(data[0])) * 0x10 + static_cast(unhex(data[1]));
+ return unhexUInt(data);
}
+/// Converts four hexadecimal digits to UInt16.
constexpr UInt16 unhex4(const char * data)
{
- return static_cast(unhex(data[0])) * 0x1000 + static_cast(unhex(data[1])) * 0x100
- + static_cast(unhex(data[2])) * 0x10 + static_cast(unhex(data[3]));
+ return unhexUInt(data);
}
-template
-constexpr TUInt unhexUInt(const char * data)
+/// Produces a binary representation of a single byte.
+inline void writeBinByte(UInt8 byte, void * out)
{
- TUInt res = 0;
- if constexpr ((sizeof(TUInt) <= 8) || ((sizeof(TUInt) % 8) != 0))
- {
- for (size_t i = 0; i < sizeof(TUInt) * 2; ++i, ++data)
- {
- res <<= 4;
- res += unhex(*data);
- }
- }
- else
- {
- for (size_t i = 0; i < sizeof(TUInt) / 8; ++i, data += 16)
- {
- res <<= 64;
- res += unhexUInt(data);
- }
- }
- return res;
+ memcpy(out, &impl::bin_byte_to_char_table[static_cast(byte) * 8], 8);
}
diff --git a/base/base/interpolate.h b/base/base/interpolate.h
index 1d4fc0b6257..4c27f70c95b 100644
--- a/base/base/interpolate.h
+++ b/base/base/interpolate.h
@@ -11,3 +11,8 @@ constexpr double interpolateExponential(double min, double max, double ratio)
assert(min > 0 && ratio >= 0 && ratio <= 1);
return min * std::pow(max / min, ratio);
}
+
+constexpr double interpolateLinear(double min, double max, double ratio)
+{
+ return std::lerp(min, max, ratio);
+}
diff --git a/base/base/move_extend.h b/base/base/move_extend.h
new file mode 100644
index 00000000000..6e5b16e037c
--- /dev/null
+++ b/base/base/move_extend.h
@@ -0,0 +1,9 @@
+#pragma once
+
+/// Extend @p to by moving elements from @p from to @p to end
+/// @return @p to iterator to first of moved elements.
+template
+typename To::iterator moveExtend(To & to, From && from)
+{
+ return to.insert(to.end(), std::make_move_iterator(from.begin()), std::make_move_iterator(from.end()));
+}
diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h
index 4a80c176829..411841e6d9f 100644
--- a/base/base/wide_integer_impl.h
+++ b/base/base/wide_integer_impl.h
@@ -27,6 +27,8 @@ using FromDoubleIntermediateType = long double;
using FromDoubleIntermediateType = boost::multiprecision::cpp_bin_float_double_extended;
#endif
+namespace CityHash_v1_0_2 { struct uint128; }
+
namespace wide
{
@@ -281,6 +283,17 @@ struct integer::_impl
}
}
+ template
+ constexpr static void wide_integer_from_cityhash_uint128(integer & self, const CityHashUInt128 & value) noexcept
+ {
+ static_assert(sizeof(item_count) >= 2);
+
+ if constexpr (std::endian::native == std::endian::little)
+ wide_integer_from_tuple_like(self, std::make_pair(value.low64, value.high64));
+ else
+ wide_integer_from_tuple_like(self, std::make_pair(value.high64, value.low64));
+ }
+
/**
* N.B. t is constructed from double, so max(t) = max(double) ~ 2^310
* the recursive call happens when t / 2^64 > 2^64, so there won't be more than 5 of them.
@@ -1036,6 +1049,8 @@ constexpr integer::integer(T rhs) noexcept
_impl::wide_integer_from_wide_integer(*this, rhs);
else if constexpr (IsTupleLike::value)
_impl::wide_integer_from_tuple_like(*this, rhs);
+ else if constexpr (std::is_same_v, CityHash_v1_0_2::uint128>)
+ _impl::wide_integer_from_cityhash_uint128(*this, rhs);
else
_impl::wide_integer_from_builtin(*this, rhs);
}
@@ -1051,6 +1066,8 @@ constexpr integer::integer(std::initializer_list il) noexcept
_impl::wide_integer_from_wide_integer(*this, *il.begin());
else if constexpr (IsTupleLike::value)
_impl::wide_integer_from_tuple_like(*this, *il.begin());
+ else if constexpr (std::is_same_v, CityHash_v1_0_2::uint128>)
+ _impl::wide_integer_from_cityhash_uint128(*this, *il.begin());
else
_impl::wide_integer_from_builtin(*this, *il.begin());
}
@@ -1088,6 +1105,8 @@ constexpr integer & integer::operator=(T rhs) noexce
{
if constexpr (IsTupleLike::value)
_impl::wide_integer_from_tuple_like(*this, rhs);
+ else if constexpr (std::is_same_v, CityHash_v1_0_2::uint128>)
+ _impl::wide_integer_from_cityhash_uint128(*this, rhs);
else
_impl::wide_integer_from_builtin(*this, rhs);
return *this;
diff --git a/base/poco/Foundation/CMakeLists.txt b/base/poco/Foundation/CMakeLists.txt
index 358f49ed055..d0dde8a51a5 100644
--- a/base/poco/Foundation/CMakeLists.txt
+++ b/base/poco/Foundation/CMakeLists.txt
@@ -87,7 +87,6 @@ set (SRCS
src/LoggingRegistry.cpp
src/LogStream.cpp
src/MD5Engine.cpp
- src/MemoryPool.cpp
src/MemoryStream.cpp
src/Message.cpp
src/Mutex.cpp
diff --git a/base/poco/Foundation/include/Poco/MemoryPool.h b/base/poco/Foundation/include/Poco/MemoryPool.h
deleted file mode 100644
index 9ab12081b5f..00000000000
--- a/base/poco/Foundation/include/Poco/MemoryPool.h
+++ /dev/null
@@ -1,116 +0,0 @@
-//
-// MemoryPool.h
-//
-// Library: Foundation
-// Package: Core
-// Module: MemoryPool
-//
-// Definition of the MemoryPool class.
-//
-// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
-// and Contributors.
-//
-// SPDX-License-Identifier: BSL-1.0
-//
-
-
-#ifndef Foundation_MemoryPool_INCLUDED
-#define Foundation_MemoryPool_INCLUDED
-
-
-#include
-#include
-#include "Poco/Foundation.h"
-#include "Poco/Mutex.h"
-
-
-namespace Poco
-{
-
-
-class Foundation_API MemoryPool
-/// A simple pool for fixed-size memory blocks.
-///
-/// The main purpose of this class is to speed-up
-/// memory allocations, as well as to reduce memory
-/// fragmentation in situations where the same blocks
-/// are allocated all over again, such as in server
-/// applications.
-///
-/// All allocated blocks are retained for future use.
-/// A limit on the number of blocks can be specified.
-/// Blocks can be preallocated.
-{
-public:
- MemoryPool(std::size_t blockSize, int preAlloc = 0, int maxAlloc = 0);
- /// Creates a MemoryPool for blocks with the given blockSize.
- /// The number of blocks given in preAlloc are preallocated.
-
- ~MemoryPool();
-
- void * get();
- /// Returns a memory block. If there are no more blocks
- /// in the pool, a new block will be allocated.
- ///
- /// If maxAlloc blocks are already allocated, an
- /// OutOfMemoryException is thrown.
-
- void release(void * ptr);
- /// Releases a memory block and returns it to the pool.
-
- std::size_t blockSize() const;
- /// Returns the block size.
-
- int allocated() const;
- /// Returns the number of allocated blocks.
-
- int available() const;
- /// Returns the number of available blocks in the pool.
-
-private:
- MemoryPool();
- MemoryPool(const MemoryPool &);
- MemoryPool & operator=(const MemoryPool &);
-
- void clear();
-
- enum
- {
- BLOCK_RESERVE = 128
- };
-
- typedef std::vector BlockVec;
-
- std::size_t _blockSize;
- int _maxAlloc;
- int _allocated;
- BlockVec _blocks;
- FastMutex _mutex;
-};
-
-
-//
-// inlines
-//
-inline std::size_t MemoryPool::blockSize() const
-{
- return _blockSize;
-}
-
-
-inline int MemoryPool::allocated() const
-{
- return _allocated;
-}
-
-
-inline int MemoryPool::available() const
-{
- return (int)_blocks.size();
-}
-
-
-} // namespace Poco
-
-
-#endif // Foundation_MemoryPool_INCLUDED
diff --git a/base/poco/Foundation/include/Poco/Message.h b/base/poco/Foundation/include/Poco/Message.h
index e8f04888ab4..282c7fb5fd1 100644
--- a/base/poco/Foundation/include/Poco/Message.h
+++ b/base/poco/Foundation/include/Poco/Message.h
@@ -67,6 +67,8 @@ public:
Message(
const std::string & source, const std::string & text, Priority prio, const char * file, int line, std::string_view fmt_str = {});
+ Message(
+ std::string && source, std::string && text, Priority prio, const char * file, int line, std::string_view fmt_str);
/// Creates a Message with the given source, text, priority,
/// source file path and line.
///
diff --git a/base/poco/Foundation/include/Poco/URI.h b/base/poco/Foundation/include/Poco/URI.h
index 1880af4ccd2..eba8109253d 100644
--- a/base/poco/Foundation/include/Poco/URI.h
+++ b/base/poco/Foundation/include/Poco/URI.h
@@ -57,7 +57,7 @@ public:
URI();
/// Creates an empty URI.
- explicit URI(const std::string & uri);
+ explicit URI(const std::string & uri, bool disable_url_encoding = false);
/// Parses an URI from the given string. Throws a
/// SyntaxException if the uri is not valid.
@@ -350,6 +350,10 @@ protected:
static const std::string ILLEGAL;
private:
+ void encodePath(std::string & encodedStr) const;
+ void decodePath(const std::string & encodedStr);
+
+
std::string _scheme;
std::string _userInfo;
std::string _host;
@@ -357,6 +361,8 @@ private:
std::string _path;
std::string _query;
std::string _fragment;
+
+ bool _disable_url_encoding = false;
};
diff --git a/base/poco/Foundation/src/MemoryPool.cpp b/base/poco/Foundation/src/MemoryPool.cpp
deleted file mode 100644
index 01c477be525..00000000000
--- a/base/poco/Foundation/src/MemoryPool.cpp
+++ /dev/null
@@ -1,105 +0,0 @@
-//
-// MemoryPool.cpp
-//
-// Library: Foundation
-// Package: Core
-// Module: MemoryPool
-//
-// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
-// and Contributors.
-//
-// SPDX-License-Identifier: BSL-1.0
-//
-
-
-#include "Poco/MemoryPool.h"
-#include "Poco/Exception.h"
-
-
-namespace Poco {
-
-
-MemoryPool::MemoryPool(std::size_t blockSize, int preAlloc, int maxAlloc):
- _blockSize(blockSize),
- _maxAlloc(maxAlloc),
- _allocated(preAlloc)
-{
- poco_assert (maxAlloc == 0 || maxAlloc >= preAlloc);
- poco_assert (preAlloc >= 0 && maxAlloc >= 0);
-
- int r = BLOCK_RESERVE;
- if (preAlloc > r)
- r = preAlloc;
- if (maxAlloc > 0 && maxAlloc < r)
- r = maxAlloc;
- _blocks.reserve(r);
-
- try
- {
- for (int i = 0; i < preAlloc; ++i)
- {
- _blocks.push_back(new char[_blockSize]);
- }
- }
- catch (...)
- {
- clear();
- throw;
- }
-}
-
-
-MemoryPool::~MemoryPool()
-{
- clear();
-}
-
-
-void MemoryPool::clear()
-{
- for (BlockVec::iterator it = _blocks.begin(); it != _blocks.end(); ++it)
- {
- delete [] *it;
- }
- _blocks.clear();
-}
-
-
-void* MemoryPool::get()
-{
- FastMutex::ScopedLock lock(_mutex);
-
- if (_blocks.empty())
- {
- if (_maxAlloc == 0 || _allocated < _maxAlloc)
- {
- ++_allocated;
- return new char[_blockSize];
- }
- else throw OutOfMemoryException("MemoryPool exhausted");
- }
- else
- {
- char* ptr = _blocks.back();
- _blocks.pop_back();
- return ptr;
- }
-}
-
-
-void MemoryPool::release(void* ptr)
-{
- FastMutex::ScopedLock lock(_mutex);
-
- try
- {
- _blocks.push_back(reinterpret_cast(ptr));
- }
- catch (...)
- {
- delete [] reinterpret_cast(ptr);
- }
-}
-
-
-} // namespace Poco
diff --git a/base/poco/Foundation/src/Message.cpp b/base/poco/Foundation/src/Message.cpp
index 663c96e47a2..54118cc0fc5 100644
--- a/base/poco/Foundation/src/Message.cpp
+++ b/base/poco/Foundation/src/Message.cpp
@@ -60,6 +60,19 @@ Message::Message(const std::string& source, const std::string& text, Priority pr
}
+Message::Message(std::string && source, std::string && text, Priority prio, const char * file, int line, std::string_view fmt_str):
+ _source(std::move(source)),
+ _text(std::move(text)),
+ _prio(prio),
+ _tid(0),
+ _file(file),
+ _line(line),
+ _pMap(0),
+ _fmt_str(fmt_str)
+{
+ init();
+}
+
Message::Message(const Message& msg):
_source(msg._source),
_text(msg._text),
diff --git a/base/poco/Foundation/src/URI.cpp b/base/poco/Foundation/src/URI.cpp
index 5543e02b279..3354c69d188 100644
--- a/base/poco/Foundation/src/URI.cpp
+++ b/base/poco/Foundation/src/URI.cpp
@@ -36,8 +36,8 @@ URI::URI():
}
-URI::URI(const std::string& uri):
- _port(0)
+URI::URI(const std::string& uri, bool decode_and_encode_path):
+ _port(0), _disable_url_encoding(decode_and_encode_path)
{
parse(uri);
}
@@ -107,7 +107,8 @@ URI::URI(const URI& uri):
_port(uri._port),
_path(uri._path),
_query(uri._query),
- _fragment(uri._fragment)
+ _fragment(uri._fragment),
+ _disable_url_encoding(uri._disable_url_encoding)
{
}
@@ -119,7 +120,8 @@ URI::URI(const URI& baseURI, const std::string& relativeURI):
_port(baseURI._port),
_path(baseURI._path),
_query(baseURI._query),
- _fragment(baseURI._fragment)
+ _fragment(baseURI._fragment),
+ _disable_url_encoding(baseURI._disable_url_encoding)
{
resolve(relativeURI);
}
@@ -151,6 +153,7 @@ URI& URI::operator = (const URI& uri)
_path = uri._path;
_query = uri._query;
_fragment = uri._fragment;
+ _disable_url_encoding = uri._disable_url_encoding;
}
return *this;
}
@@ -181,6 +184,7 @@ void URI::swap(URI& uri)
std::swap(_path, uri._path);
std::swap(_query, uri._query);
std::swap(_fragment, uri._fragment);
+ std::swap(_disable_url_encoding, uri._disable_url_encoding);
}
@@ -201,7 +205,7 @@ std::string URI::toString() const
std::string uri;
if (isRelative())
{
- encode(_path, RESERVED_PATH, uri);
+ encodePath(uri);
}
else
{
@@ -217,7 +221,7 @@ std::string URI::toString() const
{
if (!auth.empty() && _path[0] != '/')
uri += '/';
- encode(_path, RESERVED_PATH, uri);
+ encodePath(uri);
}
else if (!_query.empty() || !_fragment.empty())
{
@@ -313,7 +317,7 @@ void URI::setAuthority(const std::string& authority)
void URI::setPath(const std::string& path)
{
_path.clear();
- decode(path, _path);
+ decodePath(path);
}
@@ -418,7 +422,7 @@ void URI::setPathEtc(const std::string& pathEtc)
std::string URI::getPathEtc() const
{
std::string pathEtc;
- encode(_path, RESERVED_PATH, pathEtc);
+ encodePath(pathEtc);
if (!_query.empty())
{
pathEtc += '?';
@@ -436,7 +440,7 @@ std::string URI::getPathEtc() const
std::string URI::getPathAndQuery() const
{
std::string pathAndQuery;
- encode(_path, RESERVED_PATH, pathAndQuery);
+ encodePath(pathAndQuery);
if (!_query.empty())
{
pathAndQuery += '?';
@@ -681,6 +685,21 @@ void URI::decode(const std::string& str, std::string& decodedStr, bool plusAsSpa
}
}
+void URI::encodePath(std::string & encodedStr) const
+{
+ if (_disable_url_encoding)
+ encodedStr = _path;
+ else
+ encode(_path, RESERVED_PATH, encodedStr);
+}
+
+void URI::decodePath(const std::string & encodedStr)
+{
+ if (_disable_url_encoding)
+ _path = encodedStr;
+ else
+ decode(encodedStr, _path);
+}
bool URI::isWellKnownPort() const
{
@@ -820,7 +839,7 @@ void URI::parsePath(std::string::const_iterator& it, const std::string::const_it
{
std::string path;
while (it != end && *it != '?' && *it != '#') path += *it++;
- decode(path, _path);
+ decodePath(path);
}
diff --git a/base/poco/Net/include/Poco/Net/HTTPBasicStreamBuf.h b/base/poco/Net/include/Poco/Net/HTTPBasicStreamBuf.h
index c4872d95353..c87719b63a4 100644
--- a/base/poco/Net/include/Poco/Net/HTTPBasicStreamBuf.h
+++ b/base/poco/Net/include/Poco/Net/HTTPBasicStreamBuf.h
@@ -19,7 +19,6 @@
#include "Poco/BufferedStreamBuf.h"
-#include "Poco/Net/HTTPBufferAllocator.h"
#include "Poco/Net/Net.h"
@@ -27,9 +26,9 @@ namespace Poco
{
namespace Net
{
+ constexpr size_t HTTP_DEFAULT_BUFFER_SIZE = 8 * 1024;
-
- typedef Poco::BasicBufferedStreamBuf, HTTPBufferAllocator> HTTPBasicStreamBuf;
+ typedef Poco::BasicBufferedStreamBuf> HTTPBasicStreamBuf;
}
diff --git a/base/poco/Net/include/Poco/Net/HTTPBufferAllocator.h b/base/poco/Net/include/Poco/Net/HTTPBufferAllocator.h
deleted file mode 100644
index 5d088e35297..00000000000
--- a/base/poco/Net/include/Poco/Net/HTTPBufferAllocator.h
+++ /dev/null
@@ -1,53 +0,0 @@
-//
-// HTTPBufferAllocator.h
-//
-// Library: Net
-// Package: HTTP
-// Module: HTTPBufferAllocator
-//
-// Definition of the HTTPBufferAllocator class.
-//
-// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
-// and Contributors.
-//
-// SPDX-License-Identifier: BSL-1.0
-//
-
-
-#ifndef Net_HTTPBufferAllocator_INCLUDED
-#define Net_HTTPBufferAllocator_INCLUDED
-
-
-#include
-#include "Poco/MemoryPool.h"
-#include "Poco/Net/Net.h"
-
-
-namespace Poco
-{
-namespace Net
-{
-
-
- class Net_API HTTPBufferAllocator
- /// A BufferAllocator for HTTP streams.
- {
- public:
- static char * allocate(std::streamsize size);
- static void deallocate(char * ptr, std::streamsize size);
-
- enum
- {
- BUFFER_SIZE = 128 * 1024
- };
-
- private:
- static Poco::MemoryPool _pool;
- };
-
-
-}
-} // namespace Poco::Net
-
-
-#endif // Net_HTTPBufferAllocator_INCLUDED
diff --git a/base/poco/Net/include/Poco/Net/HTTPChunkedStream.h b/base/poco/Net/include/Poco/Net/HTTPChunkedStream.h
index 47987b18817..5f4729c9278 100644
--- a/base/poco/Net/include/Poco/Net/HTTPChunkedStream.h
+++ b/base/poco/Net/include/Poco/Net/HTTPChunkedStream.h
@@ -21,7 +21,6 @@
#include
#include
#include
-#include "Poco/MemoryPool.h"
#include "Poco/Net/HTTPBasicStreamBuf.h"
#include "Poco/Net/Net.h"
@@ -80,12 +79,6 @@ namespace Net
public:
HTTPChunkedInputStream(HTTPSession & session);
~HTTPChunkedInputStream();
-
- void * operator new(std::size_t size);
- void operator delete(void * ptr);
-
- private:
- static Poco::MemoryPool _pool;
};
@@ -95,12 +88,6 @@ namespace Net
public:
HTTPChunkedOutputStream(HTTPSession & session);
~HTTPChunkedOutputStream();
-
- void * operator new(std::size_t size);
- void operator delete(void * ptr);
-
- private:
- static Poco::MemoryPool _pool;
};
diff --git a/base/poco/Net/include/Poco/Net/HTTPClientSession.h b/base/poco/Net/include/Poco/Net/HTTPClientSession.h
index d495d662f75..167a06eb7ff 100644
--- a/base/poco/Net/include/Poco/Net/HTTPClientSession.h
+++ b/base/poco/Net/include/Poco/Net/HTTPClientSession.h
@@ -306,7 +306,7 @@ namespace Net
DEFAULT_KEEP_ALIVE_TIMEOUT = 8
};
- void reconnect();
+ virtual void reconnect();
/// Connects the underlying socket to the HTTP server.
int write(const char * buffer, std::streamsize length);
diff --git a/base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h b/base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h
index 4de211fdb92..2f4df102605 100644
--- a/base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h
+++ b/base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h
@@ -78,12 +78,6 @@ namespace Net
public:
HTTPFixedLengthInputStream(HTTPSession & session, HTTPFixedLengthStreamBuf::ContentLength length);
~HTTPFixedLengthInputStream();
-
- void * operator new(std::size_t size);
- void operator delete(void * ptr);
-
- private:
- static Poco::MemoryPool _pool;
};
@@ -93,12 +87,6 @@ namespace Net
public:
HTTPFixedLengthOutputStream(HTTPSession & session, HTTPFixedLengthStreamBuf::ContentLength length);
~HTTPFixedLengthOutputStream();
-
- void * operator new(std::size_t size);
- void operator delete(void * ptr);
-
- private:
- static Poco::MemoryPool _pool;
};
diff --git a/base/poco/Net/include/Poco/Net/HTTPHeaderStream.h b/base/poco/Net/include/Poco/Net/HTTPHeaderStream.h
index bcfca984d8b..cf1a6dba2e6 100644
--- a/base/poco/Net/include/Poco/Net/HTTPHeaderStream.h
+++ b/base/poco/Net/include/Poco/Net/HTTPHeaderStream.h
@@ -21,7 +21,6 @@
#include
#include
#include
-#include "Poco/MemoryPool.h"
#include "Poco/Net/HTTPBasicStreamBuf.h"
#include "Poco/Net/Net.h"
@@ -74,12 +73,6 @@ namespace Net
public:
HTTPHeaderInputStream(HTTPSession & session);
~HTTPHeaderInputStream();
-
- void * operator new(std::size_t size);
- void operator delete(void * ptr);
-
- private:
- static Poco::MemoryPool _pool;
};
@@ -89,12 +82,6 @@ namespace Net
public:
HTTPHeaderOutputStream(HTTPSession & session);
~HTTPHeaderOutputStream();
-
- void * operator new(std::size_t size);
- void operator delete(void * ptr);
-
- private:
- static Poco::MemoryPool _pool;
};
diff --git a/base/poco/Net/include/Poco/Net/HTTPSession.h b/base/poco/Net/include/Poco/Net/HTTPSession.h
index d0045025f5f..934b34be5d5 100644
--- a/base/poco/Net/include/Poco/Net/HTTPSession.h
+++ b/base/poco/Net/include/Poco/Net/HTTPSession.h
@@ -192,7 +192,7 @@ namespace Net
HTTPSession & operator=(const HTTPSession &);
StreamSocket _socket;
- char * _pBuffer;
+ std::unique_ptr _pBuffer;
char * _pCurrent;
char * _pEnd;
bool _keepAlive;
diff --git a/base/poco/Net/include/Poco/Net/HTTPStream.h b/base/poco/Net/include/Poco/Net/HTTPStream.h
index 0197bc62eb2..48502347b2c 100644
--- a/base/poco/Net/include/Poco/Net/HTTPStream.h
+++ b/base/poco/Net/include/Poco/Net/HTTPStream.h
@@ -21,7 +21,6 @@
#include
#include
#include
-#include "Poco/MemoryPool.h"
#include "Poco/Net/HTTPBasicStreamBuf.h"
#include "Poco/Net/Net.h"
@@ -75,12 +74,6 @@ namespace Net
public:
HTTPInputStream(HTTPSession & session);
~HTTPInputStream();
-
- void * operator new(std::size_t size);
- void operator delete(void * ptr);
-
- private:
- static Poco::MemoryPool _pool;
};
@@ -90,12 +83,6 @@ namespace Net
public:
HTTPOutputStream(HTTPSession & session);
~HTTPOutputStream();
-
- void * operator new(std::size_t size);
- void operator delete(void * ptr);
-
- private:
- static Poco::MemoryPool _pool;
};
diff --git a/base/poco/Net/src/HTTPBufferAllocator.cpp b/base/poco/Net/src/HTTPBufferAllocator.cpp
deleted file mode 100644
index 2944e2a6121..00000000000
--- a/base/poco/Net/src/HTTPBufferAllocator.cpp
+++ /dev/null
@@ -1,44 +0,0 @@
-//
-// HTTPBufferAllocator.cpp
-//
-// Library: Net
-// Package: HTTP
-// Module: HTTPBufferAllocator
-//
-// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
-// and Contributors.
-//
-// SPDX-License-Identifier: BSL-1.0
-//
-
-
-#include "Poco/Net/HTTPBufferAllocator.h"
-
-
-using Poco::MemoryPool;
-
-
-namespace Poco {
-namespace Net {
-
-
-MemoryPool HTTPBufferAllocator::_pool(HTTPBufferAllocator::BUFFER_SIZE, 16);
-
-
-char* HTTPBufferAllocator::allocate(std::streamsize size)
-{
- poco_assert_dbg (size == BUFFER_SIZE);
-
- return reinterpret_cast(_pool.get());
-}
-
-
-void HTTPBufferAllocator::deallocate(char* ptr, std::streamsize size)
-{
- poco_assert_dbg (size == BUFFER_SIZE);
-
- _pool.release(ptr);
-}
-
-
-} } // namespace Poco::Net
diff --git a/base/poco/Net/src/HTTPChunkedStream.cpp b/base/poco/Net/src/HTTPChunkedStream.cpp
index f2f79da590b..376e3f55492 100644
--- a/base/poco/Net/src/HTTPChunkedStream.cpp
+++ b/base/poco/Net/src/HTTPChunkedStream.cpp
@@ -34,7 +34,7 @@ namespace Net {
HTTPChunkedStreamBuf::HTTPChunkedStreamBuf(HTTPSession& session, openmode mode):
- HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
+ HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
_session(session),
_mode(mode),
_chunk(0)
@@ -181,10 +181,6 @@ HTTPChunkedStreamBuf* HTTPChunkedIOS::rdbuf()
// HTTPChunkedInputStream
//
-
-Poco::MemoryPool HTTPChunkedInputStream::_pool(sizeof(HTTPChunkedInputStream));
-
-
HTTPChunkedInputStream::HTTPChunkedInputStream(HTTPSession& session):
HTTPChunkedIOS(session, std::ios::in),
std::istream(&_buf)
@@ -196,34 +192,10 @@ HTTPChunkedInputStream::~HTTPChunkedInputStream()
{
}
-
-void* HTTPChunkedInputStream::operator new(std::size_t size)
-{
- return _pool.get();
-}
-
-
-void HTTPChunkedInputStream::operator delete(void* ptr)
-{
- try
- {
- _pool.release(ptr);
- }
- catch (...)
- {
- poco_unexpected();
- }
-}
-
-
//
// HTTPChunkedOutputStream
//
-
-Poco::MemoryPool HTTPChunkedOutputStream::_pool(sizeof(HTTPChunkedOutputStream));
-
-
HTTPChunkedOutputStream::HTTPChunkedOutputStream(HTTPSession& session):
HTTPChunkedIOS(session, std::ios::out),
std::ostream(&_buf)
@@ -235,24 +207,4 @@ HTTPChunkedOutputStream::~HTTPChunkedOutputStream()
{
}
-
-void* HTTPChunkedOutputStream::operator new(std::size_t size)
-{
- return _pool.get();
-}
-
-
-void HTTPChunkedOutputStream::operator delete(void* ptr)
-{
- try
- {
- _pool.release(ptr);
- }
- catch (...)
- {
- poco_unexpected();
- }
-}
-
-
} } // namespace Poco::Net
diff --git a/base/poco/Net/src/HTTPFixedLengthStream.cpp b/base/poco/Net/src/HTTPFixedLengthStream.cpp
index d19f6122ee1..fd77ff71cd9 100644
--- a/base/poco/Net/src/HTTPFixedLengthStream.cpp
+++ b/base/poco/Net/src/HTTPFixedLengthStream.cpp
@@ -30,7 +30,7 @@ namespace Net {
HTTPFixedLengthStreamBuf::HTTPFixedLengthStreamBuf(HTTPSession& session, ContentLength length, openmode mode):
- HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
+ HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
_session(session),
_length(length),
_count(0)
@@ -109,9 +109,6 @@ HTTPFixedLengthStreamBuf* HTTPFixedLengthIOS::rdbuf()
//
-Poco::MemoryPool HTTPFixedLengthInputStream::_pool(sizeof(HTTPFixedLengthInputStream));
-
-
HTTPFixedLengthInputStream::HTTPFixedLengthInputStream(HTTPSession& session, HTTPFixedLengthStreamBuf::ContentLength length):
HTTPFixedLengthIOS(session, length, std::ios::in),
std::istream(&_buf)
@@ -124,33 +121,10 @@ HTTPFixedLengthInputStream::~HTTPFixedLengthInputStream()
}
-void* HTTPFixedLengthInputStream::operator new(std::size_t size)
-{
- return _pool.get();
-}
-
-
-void HTTPFixedLengthInputStream::operator delete(void* ptr)
-{
- try
- {
- _pool.release(ptr);
- }
- catch (...)
- {
- poco_unexpected();
- }
-}
-
-
//
// HTTPFixedLengthOutputStream
//
-
-Poco::MemoryPool HTTPFixedLengthOutputStream::_pool(sizeof(HTTPFixedLengthOutputStream));
-
-
HTTPFixedLengthOutputStream::HTTPFixedLengthOutputStream(HTTPSession& session, HTTPFixedLengthStreamBuf::ContentLength length):
HTTPFixedLengthIOS(session, length, std::ios::out),
std::ostream(&_buf)
@@ -163,23 +137,4 @@ HTTPFixedLengthOutputStream::~HTTPFixedLengthOutputStream()
}
-void* HTTPFixedLengthOutputStream::operator new(std::size_t size)
-{
- return _pool.get();
-}
-
-
-void HTTPFixedLengthOutputStream::operator delete(void* ptr)
-{
- try
- {
- _pool.release(ptr);
- }
- catch (...)
- {
- poco_unexpected();
- }
-}
-
-
} } // namespace Poco::Net
diff --git a/base/poco/Net/src/HTTPHeaderStream.cpp b/base/poco/Net/src/HTTPHeaderStream.cpp
index 8e0091fcbe3..39b9007062d 100644
--- a/base/poco/Net/src/HTTPHeaderStream.cpp
+++ b/base/poco/Net/src/HTTPHeaderStream.cpp
@@ -26,7 +26,7 @@ namespace Net {
HTTPHeaderStreamBuf::HTTPHeaderStreamBuf(HTTPSession& session, openmode mode):
- HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
+ HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
_session(session),
_end(false)
{
@@ -101,10 +101,6 @@ HTTPHeaderStreamBuf* HTTPHeaderIOS::rdbuf()
// HTTPHeaderInputStream
//
-
-Poco::MemoryPool HTTPHeaderInputStream::_pool(sizeof(HTTPHeaderInputStream));
-
-
HTTPHeaderInputStream::HTTPHeaderInputStream(HTTPSession& session):
HTTPHeaderIOS(session, std::ios::in),
std::istream(&_buf)
@@ -116,34 +112,10 @@ HTTPHeaderInputStream::~HTTPHeaderInputStream()
{
}
-
-void* HTTPHeaderInputStream::operator new(std::size_t size)
-{
- return _pool.get();
-}
-
-
-void HTTPHeaderInputStream::operator delete(void* ptr)
-{
- try
- {
- _pool.release(ptr);
- }
- catch (...)
- {
- poco_unexpected();
- }
-}
-
-
//
// HTTPHeaderOutputStream
//
-
-Poco::MemoryPool HTTPHeaderOutputStream::_pool(sizeof(HTTPHeaderOutputStream));
-
-
HTTPHeaderOutputStream::HTTPHeaderOutputStream(HTTPSession& session):
HTTPHeaderIOS(session, std::ios::out),
std::ostream(&_buf)
@@ -155,24 +127,4 @@ HTTPHeaderOutputStream::~HTTPHeaderOutputStream()
{
}
-
-void* HTTPHeaderOutputStream::operator new(std::size_t size)
-{
- return _pool.get();
-}
-
-
-void HTTPHeaderOutputStream::operator delete(void* ptr)
-{
- try
- {
- _pool.release(ptr);
- }
- catch (...)
- {
- poco_unexpected();
- }
-}
-
-
} } // namespace Poco::Net
diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp
index cb6fdc25e9a..d2663baaf9f 100644
--- a/base/poco/Net/src/HTTPSession.cpp
+++ b/base/poco/Net/src/HTTPSession.cpp
@@ -13,8 +13,8 @@
#include "Poco/Net/HTTPSession.h"
-#include "Poco/Net/HTTPBufferAllocator.h"
#include "Poco/Net/NetException.h"
+#include "Poco/Net/HTTPBasicStreamBuf.h"
#include
@@ -68,14 +68,6 @@ HTTPSession::HTTPSession(const StreamSocket& socket, bool keepAlive):
HTTPSession::~HTTPSession()
{
- try
- {
- if (_pBuffer) HTTPBufferAllocator::deallocate(_pBuffer, HTTPBufferAllocator::BUFFER_SIZE);
- }
- catch (...)
- {
- poco_unexpected();
- }
try
{
close();
@@ -177,10 +169,10 @@ void HTTPSession::refill()
{
if (!_pBuffer)
{
- _pBuffer = HTTPBufferAllocator::allocate(HTTPBufferAllocator::BUFFER_SIZE);
+ _pBuffer = std::make_unique(HTTP_DEFAULT_BUFFER_SIZE);
}
- _pCurrent = _pEnd = _pBuffer;
- int n = receive(_pBuffer, HTTPBufferAllocator::BUFFER_SIZE);
+ _pCurrent = _pEnd = _pBuffer.get();
+ int n = receive(_pBuffer.get(), HTTP_DEFAULT_BUFFER_SIZE);
_pEnd += n;
}
@@ -199,7 +191,7 @@ void HTTPSession::connect(const SocketAddress& address)
_socket.setNoDelay(true);
// There may be leftover data from a previous (failed) request in the buffer,
// so we clear it.
- _pCurrent = _pEnd = _pBuffer;
+ _pCurrent = _pEnd = _pBuffer.get();
}
diff --git a/base/poco/Net/src/HTTPStream.cpp b/base/poco/Net/src/HTTPStream.cpp
index 4acb881c4f3..c2f27600569 100644
--- a/base/poco/Net/src/HTTPStream.cpp
+++ b/base/poco/Net/src/HTTPStream.cpp
@@ -26,7 +26,7 @@ namespace Net {
HTTPStreamBuf::HTTPStreamBuf(HTTPSession& session, openmode mode):
- HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
+ HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
_session(session),
_mode(mode)
{
@@ -96,10 +96,6 @@ HTTPStreamBuf* HTTPIOS::rdbuf()
// HTTPInputStream
//
-
-Poco::MemoryPool HTTPInputStream::_pool(sizeof(HTTPInputStream));
-
-
HTTPInputStream::HTTPInputStream(HTTPSession& session):
HTTPIOS(session, std::ios::in),
std::istream(&_buf)
@@ -112,33 +108,11 @@ HTTPInputStream::~HTTPInputStream()
}
-void* HTTPInputStream::operator new(std::size_t size)
-{
- return _pool.get();
-}
-
-
-void HTTPInputStream::operator delete(void* ptr)
-{
- try
- {
- _pool.release(ptr);
- }
- catch (...)
- {
- poco_unexpected();
- }
-}
-
-
//
// HTTPOutputStream
//
-Poco::MemoryPool HTTPOutputStream::_pool(sizeof(HTTPOutputStream));
-
-
HTTPOutputStream::HTTPOutputStream(HTTPSession& session):
HTTPIOS(session, std::ios::out),
std::ostream(&_buf)
@@ -150,24 +124,4 @@ HTTPOutputStream::~HTTPOutputStream()
{
}
-
-void* HTTPOutputStream::operator new(std::size_t size)
-{
- return _pool.get();
-}
-
-
-void HTTPOutputStream::operator delete(void* ptr)
-{
- try
- {
- _pool.release(ptr);
- }
- catch (...)
- {
- poco_unexpected();
- }
-}
-
-
} } // namespace Poco::Net
diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/ConsoleCertificateHandler.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/ConsoleCertificateHandler.h
deleted file mode 100644
index 8e09b6f18ae..00000000000
--- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/ConsoleCertificateHandler.h
+++ /dev/null
@@ -1,53 +0,0 @@
-//
-// ConsoleCertificateHandler.h
-//
-// Library: NetSSL_OpenSSL
-// Package: SSLCore
-// Module: ConsoleCertificateHandler
-//
-// Definition of the ConsoleCertificateHandler class.
-//
-// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH.
-// and Contributors.
-//
-// SPDX-License-Identifier: BSL-1.0
-//
-
-
-#ifndef NetSSL_ConsoleCertificateHandler_INCLUDED
-#define NetSSL_ConsoleCertificateHandler_INCLUDED
-
-
-#include "Poco/Net/InvalidCertificateHandler.h"
-#include "Poco/Net/NetSSL.h"
-
-
-namespace Poco
-{
-namespace Net
-{
-
-
- class NetSSL_API ConsoleCertificateHandler : public InvalidCertificateHandler
- /// A ConsoleCertificateHandler is invoked whenever an error occurs verifying the certificate.
- ///
- /// The certificate is printed to stdout and the user is asked via console if he wants to accept it.
- {
- public:
- ConsoleCertificateHandler(bool handleErrorsOnServerSide);
- /// Creates the ConsoleCertificateHandler.
-
- virtual ~ConsoleCertificateHandler();
- /// Destroys the ConsoleCertificateHandler.
-
- void onInvalidCertificate(const void * pSender, VerificationErrorArgs & errorCert);
- /// Prints the certificate to stdout and waits for user input on the console
- /// to decide if a certificate should be accepted/rejected.
- };
-
-
-}
-} // namespace Poco::Net
-
-
-#endif // NetSSL_ConsoleCertificateHandler_INCLUDED
diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h
index a4fde26286e..21a1ed685e5 100644
--- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h
+++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h
@@ -85,7 +85,7 @@ namespace Net
///
///
///
- /// ConsoleCertificateHandler
+ /// RejectCertificateHandler
///
/// true|false
/// someString
@@ -186,7 +186,7 @@ namespace Net
///
/// Valid initialization code would be:
/// SharedPtr pConsoleHandler = new KeyConsoleHandler;
- /// SharedPtr pInvalidCertHandler = new ConsoleCertificateHandler;
+ /// SharedPtr pInvalidCertHandler = new RejectCertificateHandler;
/// Context::Ptr pContext = new Context(Context::SERVER_USE, "any.pem", "any.pem", "rootcert.pem", Context::VERIFY_RELAXED, 9, false, "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH");
/// SSLManager::instance().initializeServer(pConsoleHandler, pInvalidCertHandler, pContext);
@@ -203,7 +203,7 @@ namespace Net
///
/// Valid initialization code would be:
/// SharedPtr pConsoleHandler = new KeyConsoleHandler;
- /// SharedPtr pInvalidCertHandler = new ConsoleCertificateHandler;
+ /// SharedPtr pInvalidCertHandler = new RejectCertificateHandler;
/// Context::Ptr pContext = new Context(Context::CLIENT_USE, "", "", "rootcert.pem", Context::VERIFY_RELAXED, 9, false, "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH");
/// SSLManager::instance().initializeClient(pConsoleHandler, pInvalidCertHandler, pContext);
diff --git a/base/poco/NetSSL_OpenSSL/src/CertificateHandlerFactoryMgr.cpp b/base/poco/NetSSL_OpenSSL/src/CertificateHandlerFactoryMgr.cpp
index a89bbea11f2..f570e2d3599 100644
--- a/base/poco/NetSSL_OpenSSL/src/CertificateHandlerFactoryMgr.cpp
+++ b/base/poco/NetSSL_OpenSSL/src/CertificateHandlerFactoryMgr.cpp
@@ -13,7 +13,6 @@
#include "Poco/Net/CertificateHandlerFactoryMgr.h"
-#include "Poco/Net/ConsoleCertificateHandler.h"
#include "Poco/Net/AcceptCertificateHandler.h"
#include "Poco/Net/RejectCertificateHandler.h"
@@ -24,7 +23,6 @@ namespace Net {
CertificateHandlerFactoryMgr::CertificateHandlerFactoryMgr()
{
- setFactory("ConsoleCertificateHandler", new CertificateHandlerFactoryImpl());
setFactory("AcceptCertificateHandler", new CertificateHandlerFactoryImpl());
setFactory("RejectCertificateHandler", new CertificateHandlerFactoryImpl());
}
diff --git a/base/poco/NetSSL_OpenSSL/src/ConsoleCertificateHandler.cpp b/base/poco/NetSSL_OpenSSL/src/ConsoleCertificateHandler.cpp
deleted file mode 100644
index db64752e70c..00000000000
--- a/base/poco/NetSSL_OpenSSL/src/ConsoleCertificateHandler.cpp
+++ /dev/null
@@ -1,53 +0,0 @@
-//
-// ConsoleCertificateHandler.cpp
-//
-// Library: NetSSL_OpenSSL
-// Package: SSLCore
-// Module: ConsoleCertificateHandler
-//
-// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH.
-// and Contributors.
-//
-// SPDX-License-Identifier: BSL-1.0
-//
-
-
-#include "Poco/Net/ConsoleCertificateHandler.h"
-#include
-
-
-namespace Poco {
-namespace Net {
-
-
-ConsoleCertificateHandler::ConsoleCertificateHandler(bool server): InvalidCertificateHandler(server)
-{
-}
-
-
-ConsoleCertificateHandler::~ConsoleCertificateHandler()
-{
-}
-
-
-void ConsoleCertificateHandler::onInvalidCertificate(const void*, VerificationErrorArgs& errorCert)
-{
- const X509Certificate& aCert = errorCert.certificate();
- std::cout << "\n";
- std::cout << "WARNING: Certificate verification failed\n";
- std::cout << "----------------------------------------\n";
- std::cout << "Issuer Name: " << aCert.issuerName() << "\n";
- std::cout << "Subject Name: " << aCert.subjectName() << "\n\n";
- std::cout << "The certificate yielded the error: " << errorCert.errorMessage() << "\n\n";
- std::cout << "The error occurred in the certificate chain at position " << errorCert.errorDepth() << "\n";
- std::cout << "Accept the certificate (y,n)? ";
- char c = 0;
- std::cin >> c;
- if (c == 'y' || c == 'Y')
- errorCert.setIgnoreError(true);
- else
- errorCert.setIgnoreError(false);
-}
-
-
-} } // namespace Poco::Net
diff --git a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp
index 82eed1a29eb..927602ca658 100644
--- a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp
+++ b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp
@@ -46,7 +46,7 @@ const std::string SSLManager::CFG_PREFER_SERVER_CIPHERS("preferServerCiphers");
const std::string SSLManager::CFG_DELEGATE_HANDLER("privateKeyPassphraseHandler.name");
const std::string SSLManager::VAL_DELEGATE_HANDLER("KeyConsoleHandler");
const std::string SSLManager::CFG_CERTIFICATE_HANDLER("invalidCertificateHandler.name");
-const std::string SSLManager::VAL_CERTIFICATE_HANDLER("ConsoleCertificateHandler");
+const std::string SSLManager::VAL_CERTIFICATE_HANDLER("RejectCertificateHandler");
const std::string SSLManager::CFG_SERVER_PREFIX("openSSL.server.");
const std::string SSLManager::CFG_CLIENT_PREFIX("openSSL.client.");
const std::string SSLManager::CFG_CACHE_SESSIONS("cacheSessions");
diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
index 015037b2de6..821b7b46855 100644
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@@ -2,11 +2,11 @@
# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54475)
+SET(VERSION_REVISION 54476)
SET(VERSION_MAJOR 23)
-SET(VERSION_MINOR 6)
+SET(VERSION_MINOR 7)
SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 2fec796e73efda10a538a03af3205ce8ffa1b2de)
-SET(VERSION_DESCRIBE v23.6.1.1-testing)
-SET(VERSION_STRING 23.6.1.1)
+SET(VERSION_GITHASH d1c7e13d08868cb04d3562dcced704dd577cb1df)
+SET(VERSION_DESCRIBE v23.7.1.1-testing)
+SET(VERSION_STRING 23.7.1.1)
# end of autochange
diff --git a/cmake/darwin/default_libs.cmake b/cmake/darwin/default_libs.cmake
index 812847e6201..42b8473cb75 100644
--- a/cmake/darwin/default_libs.cmake
+++ b/cmake/darwin/default_libs.cmake
@@ -15,6 +15,7 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET 10.15)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
+include (cmake/unwind.cmake)
include (cmake/cxx.cmake)
link_libraries(global-group)
diff --git a/cmake/embed_binary.cmake b/cmake/embed_binary.cmake
deleted file mode 100644
index e5428c24939..00000000000
--- a/cmake/embed_binary.cmake
+++ /dev/null
@@ -1,58 +0,0 @@
-# Embed a set of resource files into a resulting object file.
-#
-# Signature: `clickhouse_embed_binaries(TARGET RESOURCE_DIR RESOURCES ...)
-#
-# This will generate a static library target named ``, which contains the contents of
-# each `` file. The files should be located in ``. defaults to
-# ${CMAKE_CURRENT_SOURCE_DIR}, and the resources may not be empty.
-#
-# Each resource will result in three symbols in the final archive, based on the name ``.
-# These are:
-# 1. `_binary__start`: Points to the start of the binary data from ``.
-# 2. `_binary__end`: Points to the end of the binary data from ``.
-# 2. `_binary__size`: Points to the size of the binary data from ``.
-#
-# `` is a normalized name derived from ``, by replacing the characters "./-" with
-# the character "_", and the character "+" with "_PLUS_". This scheme is similar to those generated
-# by `ld -r -b binary`, and matches the expectations in `./base/common/getResource.cpp`.
-macro(clickhouse_embed_binaries)
- set(one_value_args TARGET RESOURCE_DIR)
- set(resources RESOURCES)
- cmake_parse_arguments(EMBED "" "${one_value_args}" ${resources} ${ARGN})
-
- if (NOT DEFINED EMBED_TARGET)
- message(FATAL_ERROR "A target name must be provided for embedding binary resources into")
- endif()
-
- if (NOT DEFINED EMBED_RESOURCE_DIR)
- set(EMBED_RESOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
- endif()
-
- list(LENGTH EMBED_RESOURCES N_RESOURCES)
- if (N_RESOURCES LESS 1)
- message(FATAL_ERROR "The list of binary resources to embed may not be empty")
- endif()
-
- add_library("${EMBED_TARGET}" STATIC)
- set_target_properties("${EMBED_TARGET}" PROPERTIES LINKER_LANGUAGE C)
-
- set(EMBED_TEMPLATE_FILE "${PROJECT_SOURCE_DIR}/programs/embed_binary.S.in")
-
- foreach(RESOURCE_FILE ${EMBED_RESOURCES})
- set(ASSEMBLY_FILE_NAME "${RESOURCE_FILE}.S")
- set(BINARY_FILE_NAME "${RESOURCE_FILE}")
-
- # Normalize the name of the resource.
- string(REGEX REPLACE "[\./-]" "_" SYMBOL_NAME "${RESOURCE_FILE}") # - must be last in regex
- string(REPLACE "+" "_PLUS_" SYMBOL_NAME "${SYMBOL_NAME}")
-
- # Generate the configured assembly file in the output directory.
- configure_file("${EMBED_TEMPLATE_FILE}" "${CMAKE_CURRENT_BINARY_DIR}/${ASSEMBLY_FILE_NAME}" @ONLY)
-
- # Set the include directory for relative paths specified for `.incbin` directive.
- set_property(SOURCE "${CMAKE_CURRENT_BINARY_DIR}/${ASSEMBLY_FILE_NAME}" APPEND PROPERTY INCLUDE_DIRECTORIES "${EMBED_RESOURCE_DIR}")
-
- target_sources("${EMBED_TARGET}" PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/${ASSEMBLY_FILE_NAME}")
- set_target_properties("${EMBED_TARGET}" PROPERTIES OBJECT_DEPENDS "${RESOURCE_FILE}")
- endforeach()
-endmacro()
diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake
index a8f105b8987..acc38b6fa2a 100644
--- a/cmake/limit_jobs.cmake
+++ b/cmake/limit_jobs.cmake
@@ -1,38 +1,39 @@
-# Usage:
-# set (MAX_COMPILER_MEMORY 2000 CACHE INTERNAL "") # In megabytes
-# set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "")
-# include (cmake/limit_jobs.cmake)
+# Limit compiler/linker job concurrency to avoid OOMs on subtrees where compilation/linking is memory-intensive.
+#
+# Usage from CMake:
+# set (MAX_COMPILER_MEMORY 2000 CACHE INTERNAL "") # megabyte
+# set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "") # megabyte
+# include (cmake/limit_jobs.cmake)
+#
+# (bigger values mean fewer jobs)
-cmake_host_system_information(RESULT TOTAL_PHYSICAL_MEMORY QUERY TOTAL_PHYSICAL_MEMORY) # Not available under freebsd
+cmake_host_system_information(RESULT TOTAL_PHYSICAL_MEMORY QUERY TOTAL_PHYSICAL_MEMORY)
cmake_host_system_information(RESULT NUMBER_OF_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES)
-# 1 if not set
-option(PARALLEL_COMPILE_JOBS "Maximum number of concurrent compilation jobs" "")
+# Set to disable the automatic job-limiting
+option(PARALLEL_COMPILE_JOBS "Maximum number of concurrent compilation jobs" OFF)
+option(PARALLEL_LINK_JOBS "Maximum number of concurrent link jobs" OFF)
-# 1 if not set
-option(PARALLEL_LINK_JOBS "Maximum number of concurrent link jobs" "")
-
-if (NOT PARALLEL_COMPILE_JOBS AND TOTAL_PHYSICAL_MEMORY AND MAX_COMPILER_MEMORY)
+if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY)
math(EXPR PARALLEL_COMPILE_JOBS ${TOTAL_PHYSICAL_MEMORY}/${MAX_COMPILER_MEMORY})
if (NOT PARALLEL_COMPILE_JOBS)
set (PARALLEL_COMPILE_JOBS 1)
endif ()
+ if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES)
+ message(WARNING "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.")
+ endif()
endif ()
-if (PARALLEL_COMPILE_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES))
- set(CMAKE_JOB_POOL_COMPILE compile_job_pool${CMAKE_CURRENT_SOURCE_DIR})
- string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_COMPILE ${CMAKE_JOB_POOL_COMPILE})
- set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_COMPILE}=${PARALLEL_COMPILE_JOBS})
-endif ()
-
-
-if (NOT PARALLEL_LINK_JOBS AND TOTAL_PHYSICAL_MEMORY AND MAX_LINKER_MEMORY)
+if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY)
math(EXPR PARALLEL_LINK_JOBS ${TOTAL_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY})
if (NOT PARALLEL_LINK_JOBS)
set (PARALLEL_LINK_JOBS 1)
endif ()
+ if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES)
+ message(WARNING "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.")
+ endif()
endif ()
# ThinLTO provides its own parallel linking
@@ -46,14 +47,16 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLE
set (PARALLEL_LINK_JOBS 2)
endif()
-if (PARALLEL_LINK_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES))
+message(STATUS "Building sub-tree with ${PARALLEL_COMPILE_JOBS} compile jobs and ${PARALLEL_LINK_JOBS} linker jobs (system: ${NUMBER_OF_LOGICAL_CORES} cores, ${TOTAL_PHYSICAL_MEMORY} MB DRAM, 'OFF' means the native core count).")
+
+if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES)
+ set(CMAKE_JOB_POOL_COMPILE compile_job_pool${CMAKE_CURRENT_SOURCE_DIR})
+ string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_COMPILE ${CMAKE_JOB_POOL_COMPILE})
+ set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_COMPILE}=${PARALLEL_COMPILE_JOBS})
+endif ()
+
+if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES)
set(CMAKE_JOB_POOL_LINK link_job_pool${CMAKE_CURRENT_SOURCE_DIR})
string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_LINK ${CMAKE_JOB_POOL_LINK})
set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_LINK}=${PARALLEL_LINK_JOBS})
endif ()
-
-if (PARALLEL_COMPILE_JOBS OR PARALLEL_LINK_JOBS)
- message(STATUS
- "${CMAKE_CURRENT_SOURCE_DIR}: Have ${TOTAL_PHYSICAL_MEMORY} megabytes of memory.
- Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS} (system has ${NUMBER_OF_LOGICAL_CORES} logical cores)")
-endif ()
diff --git a/cmake/target.cmake b/cmake/target.cmake
index 5ef45576fb7..ffab08f1103 100644
--- a/cmake/target.cmake
+++ b/cmake/target.cmake
@@ -33,6 +33,18 @@ if (CMAKE_CROSSCOMPILING)
elseif (ARCH_PPC64LE)
set (ENABLE_GRPC OFF CACHE INTERNAL "")
set (ENABLE_SENTRY OFF CACHE INTERNAL "")
+ elseif (ARCH_RISCV64)
+ # RISC-V support is preliminary
+ set (GLIBC_COMPATIBILITY OFF CACHE INTERNAL "")
+ set (ENABLE_LDAP OFF CACHE INTERNAL "")
+ set (OPENSSL_NO_ASM ON CACHE INTERNAL "")
+ set (ENABLE_JEMALLOC ON CACHE INTERNAL "")
+ set (ENABLE_PARQUET OFF CACHE INTERNAL "")
+ set (ENABLE_GRPC OFF CACHE INTERNAL "")
+ set (ENABLE_HDFS OFF CACHE INTERNAL "")
+ set (ENABLE_MYSQL OFF CACHE INTERNAL "")
+ # It might be ok, but we need to update 'sysroot'
+ set (ENABLE_RUST OFF CACHE INTERNAL "")
elseif (ARCH_S390X)
set (ENABLE_GRPC OFF CACHE INTERNAL "")
set (ENABLE_SENTRY OFF CACHE INTERNAL "")
diff --git a/cmake/unwind.cmake b/cmake/unwind.cmake
index c9f5f30a5d6..84e4f01b752 100644
--- a/cmake/unwind.cmake
+++ b/cmake/unwind.cmake
@@ -1,13 +1 @@
-option (USE_UNWIND "Enable libunwind (better stacktraces)" ${ENABLE_LIBRARIES})
-
-if (USE_UNWIND)
- add_subdirectory(contrib/libunwind-cmake)
- set (UNWIND_LIBRARIES unwind)
- set (EXCEPTION_HANDLING_LIBRARY ${UNWIND_LIBRARIES})
-
- message (STATUS "Using libunwind: ${UNWIND_LIBRARIES}")
-else ()
- set (EXCEPTION_HANDLING_LIBRARY gcc_eh)
-endif ()
-
-message (STATUS "Using exception handler: ${EXCEPTION_HANDLING_LIBRARY}")
+add_subdirectory(contrib/libunwind-cmake)
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 4a4ff9982ea..fdf6e60e58f 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -146,7 +146,7 @@ add_contrib (amqpcpp-cmake AMQP-CPP) # requires: libuv
add_contrib (cassandra-cmake cassandra) # requires: libuv
if (NOT OS_DARWIN)
add_contrib (curl-cmake curl)
- add_contrib (azure-cmake azure)
+ add_contrib (azure-cmake azure) # requires: curl
add_contrib (sentry-native-cmake sentry-native) # requires: curl
endif()
add_contrib (fmtlib-cmake fmtlib)
@@ -157,21 +157,20 @@ add_contrib (librdkafka-cmake librdkafka) # requires: libgsasl
add_contrib (nats-io-cmake nats-io)
add_contrib (isa-l-cmake isa-l)
add_contrib (libhdfs3-cmake libhdfs3) # requires: google-protobuf, krb5, isa-l
-add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3
+add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift, avro, arrow, libhdfs3
add_contrib (cppkafka-cmake cppkafka)
add_contrib (libpqxx-cmake libpqxx)
add_contrib (libpq-cmake libpq)
add_contrib (nuraft-cmake NuRaft)
add_contrib (fast_float-cmake fast_float)
add_contrib (datasketches-cpp-cmake datasketches-cpp)
-add_contrib (hashidsxx-cmake hashidsxx)
+add_contrib (incbin-cmake incbin)
option(ENABLE_NLP "Enable NLP functions support" ${ENABLE_LIBRARIES})
if (ENABLE_NLP)
add_contrib (libstemmer-c-cmake libstemmer_c)
add_contrib (wordnet-blast-cmake wordnet-blast)
add_contrib (lemmagen-c-cmake lemmagen-c)
- add_contrib (nlp-data-cmake nlp-data)
add_contrib (cld2-cmake cld2)
endif()
diff --git a/contrib/NuRaft b/contrib/NuRaft
index 491eaf592d9..eb1572129c7 160000
--- a/contrib/NuRaft
+++ b/contrib/NuRaft
@@ -1 +1 @@
-Subproject commit 491eaf592d950e0e37accbe8b3f217e068c9fecf
+Subproject commit eb1572129c71beb2156dcdaadc3fb136954aed96
diff --git a/contrib/abseil-cpp-cmake/CMakeLists.txt b/contrib/abseil-cpp-cmake/CMakeLists.txt
index 4c31ecfc599..d64b6036611 100644
--- a/contrib/abseil-cpp-cmake/CMakeLists.txt
+++ b/contrib/abseil-cpp-cmake/CMakeLists.txt
@@ -17,3 +17,17 @@ get_target_property(FLAT_HASH_SET_INCLUDE_DIR absl::flat_hash_set INTERFACE_INCL
target_include_directories (_abseil_swiss_tables SYSTEM BEFORE INTERFACE ${FLAT_HASH_SET_INCLUDE_DIR})
add_library(ch_contrib::abseil_swiss_tables ALIAS _abseil_swiss_tables)
+
+set(ABSL_FORMAT_SRC
+ ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/arg.cc
+ ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/bind.cc
+ ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/extension.cc
+ ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/float_conversion.cc
+ ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/output.cc
+ ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/parser.cc
+)
+
+add_library(_abseil_str_format ${ABSL_FORMAT_SRC})
+target_include_directories(_abseil_str_format PUBLIC ${ABSL_ROOT_DIR})
+
+add_library(ch_contrib::abseil_str_format ALIAS _abseil_str_format)
diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt
index 16198887075..e3ea0381595 100644
--- a/contrib/arrow-cmake/CMakeLists.txt
+++ b/contrib/arrow-cmake/CMakeLists.txt
@@ -31,12 +31,12 @@ endif()
set (CMAKE_CXX_STANDARD 17)
-set(ARROW_VERSION "6.0.1")
+set(ARROW_VERSION "11.0.0")
string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}")
-set(ARROW_VERSION_MAJOR "6")
+set(ARROW_VERSION_MAJOR "11")
set(ARROW_VERSION_MINOR "0")
-set(ARROW_VERSION_PATCH "1")
+set(ARROW_VERSION_PATCH "0")
if(ARROW_VERSION_MAJOR STREQUAL "0")
# Arrow 0.x.y => SO version is "x", full SO version is "x.y.0"
@@ -116,43 +116,79 @@ configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/A
# ARROW_ORC + adapters/orc/CMakefiles
set(ORC_SRCS
"${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h"
- "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.cc"
- "${ORC_SOURCE_SRC_DIR}/sargs/Literal.cc"
- "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.cc"
- "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.cc"
- "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.cc"
- "${ORC_SOURCE_SRC_DIR}/sargs/TruthValue.cc"
- "${ORC_SOURCE_SRC_DIR}/Exceptions.cc"
- "${ORC_SOURCE_SRC_DIR}/OrcFile.cc"
- "${ORC_SOURCE_SRC_DIR}/Reader.cc"
+ "${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc"
+ "${ORC_SOURCE_SRC_DIR}/Adaptor.cc"
+ "${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in"
+ "${ORC_SOURCE_SRC_DIR}/BlockBuffer.cc"
+ "${ORC_SOURCE_SRC_DIR}/BlockBuffer.hh"
+ "${ORC_SOURCE_SRC_DIR}/BloomFilter.cc"
+ "${ORC_SOURCE_SRC_DIR}/BloomFilter.hh"
+ "${ORC_SOURCE_SRC_DIR}/Bpacking.hh"
+ "${ORC_SOURCE_SRC_DIR}/BpackingDefault.cc"
+ "${ORC_SOURCE_SRC_DIR}/BpackingDefault.hh"
"${ORC_SOURCE_SRC_DIR}/ByteRLE.cc"
+ "${ORC_SOURCE_SRC_DIR}/ByteRLE.hh"
+ "${ORC_SOURCE_SRC_DIR}/CMakeLists.txt"
"${ORC_SOURCE_SRC_DIR}/ColumnPrinter.cc"
"${ORC_SOURCE_SRC_DIR}/ColumnReader.cc"
+ "${ORC_SOURCE_SRC_DIR}/ColumnReader.hh"
"${ORC_SOURCE_SRC_DIR}/ColumnWriter.cc"
+ "${ORC_SOURCE_SRC_DIR}/ColumnWriter.hh"
"${ORC_SOURCE_SRC_DIR}/Common.cc"
"${ORC_SOURCE_SRC_DIR}/Compression.cc"
+ "${ORC_SOURCE_SRC_DIR}/Compression.hh"
+ "${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.cc"
+ "${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.hh"
+ "${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.cc"
+ "${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.hh"
+ "${ORC_SOURCE_SRC_DIR}/Dispatch.hh"
+ "${ORC_SOURCE_SRC_DIR}/Exceptions.cc"
"${ORC_SOURCE_SRC_DIR}/Int128.cc"
"${ORC_SOURCE_SRC_DIR}/LzoDecompressor.cc"
+ "${ORC_SOURCE_SRC_DIR}/LzoDecompressor.hh"
"${ORC_SOURCE_SRC_DIR}/MemoryPool.cc"
+ "${ORC_SOURCE_SRC_DIR}/Murmur3.cc"
+ "${ORC_SOURCE_SRC_DIR}/Murmur3.hh"
+ "${ORC_SOURCE_SRC_DIR}/Options.hh"
+ "${ORC_SOURCE_SRC_DIR}/OrcFile.cc"
"${ORC_SOURCE_SRC_DIR}/RLE.cc"
+ "${ORC_SOURCE_SRC_DIR}/RLE.hh"
+ "${ORC_SOURCE_SRC_DIR}/RLEV2Util.cc"
+ "${ORC_SOURCE_SRC_DIR}/RLEV2Util.hh"
"${ORC_SOURCE_SRC_DIR}/RLEv1.cc"
+ "${ORC_SOURCE_SRC_DIR}/RLEv1.hh"
+ "${ORC_SOURCE_SRC_DIR}/RLEv2.hh"
+ "${ORC_SOURCE_SRC_DIR}/Reader.cc"
+ "${ORC_SOURCE_SRC_DIR}/Reader.hh"
"${ORC_SOURCE_SRC_DIR}/RleDecoderV2.cc"
"${ORC_SOURCE_SRC_DIR}/RleEncoderV2.cc"
- "${ORC_SOURCE_SRC_DIR}/RLEV2Util.cc"
+ "${ORC_SOURCE_SRC_DIR}/SchemaEvolution.cc"
+ "${ORC_SOURCE_SRC_DIR}/SchemaEvolution.hh"
"${ORC_SOURCE_SRC_DIR}/Statistics.cc"
+ "${ORC_SOURCE_SRC_DIR}/Statistics.hh"
"${ORC_SOURCE_SRC_DIR}/StripeStream.cc"
+ "${ORC_SOURCE_SRC_DIR}/StripeStream.hh"
"${ORC_SOURCE_SRC_DIR}/Timezone.cc"
+ "${ORC_SOURCE_SRC_DIR}/Timezone.hh"
"${ORC_SOURCE_SRC_DIR}/TypeImpl.cc"
+ "${ORC_SOURCE_SRC_DIR}/TypeImpl.hh"
+ "${ORC_SOURCE_SRC_DIR}/Utils.hh"
"${ORC_SOURCE_SRC_DIR}/Vector.cc"
"${ORC_SOURCE_SRC_DIR}/Writer.cc"
- "${ORC_SOURCE_SRC_DIR}/Adaptor.cc"
- "${ORC_SOURCE_SRC_DIR}/BloomFilter.cc"
- "${ORC_SOURCE_SRC_DIR}/Murmur3.cc"
- "${ORC_SOURCE_SRC_DIR}/BlockBuffer.cc"
- "${ORC_SOURCE_SRC_DIR}/wrap/orc-proto-wrapper.cc"
"${ORC_SOURCE_SRC_DIR}/io/InputStream.cc"
+ "${ORC_SOURCE_SRC_DIR}/io/InputStream.hh"
"${ORC_SOURCE_SRC_DIR}/io/OutputStream.cc"
- "${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc"
+ "${ORC_SOURCE_SRC_DIR}/io/OutputStream.hh"
+ "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.cc"
+ "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.hh"
+ "${ORC_SOURCE_SRC_DIR}/sargs/Literal.cc"
+ "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.cc"
+ "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.hh"
+ "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.cc"
+ "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.hh"
+ "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.cc"
+ "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.hh"
+ "${ORC_SOURCE_SRC_DIR}/sargs/TruthValue.cc"
)
add_library(_orc ${ORC_SRCS})
@@ -466,9 +502,10 @@ target_include_directories(_parquet SYSTEM BEFORE
"${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src"
"${CMAKE_CURRENT_SOURCE_DIR}/cpp/src")
target_link_libraries(_parquet
- PUBLIC _arrow
- PRIVATE
+ PUBLIC
+ _arrow
ch_contrib::thrift
+ PRIVATE
boost::headers_only
boost::regex
OpenSSL::Crypto OpenSSL::SSL)
@@ -478,6 +515,10 @@ if (SANITIZE STREQUAL "undefined")
target_compile_options(_arrow PRIVATE -fno-sanitize=undefined)
endif ()
+# Define Thrift version for parquet (we use 0.16.0)
+add_definitions(-DPARQUET_THRIFT_VERSION_MAJOR=0)
+add_definitions(-DPARQUET_THRIFT_VERSION_MINOR=16)
+
# === tools
set(TOOLS_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/tools/parquet")
diff --git a/contrib/azure-cmake/CMakeLists.txt b/contrib/azure-cmake/CMakeLists.txt
index 887122e7653..7aba81259d3 100644
--- a/contrib/azure-cmake/CMakeLists.txt
+++ b/contrib/azure-cmake/CMakeLists.txt
@@ -1,6 +1,6 @@
option (ENABLE_AZURE_BLOB_STORAGE "Enable Azure blob storage" ${ENABLE_LIBRARIES})
-if (NOT ENABLE_AZURE_BLOB_STORAGE OR BUILD_STANDALONE_KEEPER OR OS_FREEBSD OR (NOT ARCH_AMD64))
+if (NOT ENABLE_AZURE_BLOB_STORAGE OR OS_FREEBSD)
message(STATUS "Not using Azure blob storage")
return()
endif()
diff --git a/contrib/cctz b/contrib/cctz
index 5e05432420f..8529bcef5cd 160000
--- a/contrib/cctz
+++ b/contrib/cctz
@@ -1 +1 @@
-Subproject commit 5e05432420f9692418e2e12aff09859e420b14a2
+Subproject commit 8529bcef5cd996b7c0f4d7475286b76b5d126c4c
diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt
index 10070fbd949..7161f743de1 100644
--- a/contrib/cctz-cmake/CMakeLists.txt
+++ b/contrib/cctz-cmake/CMakeLists.txt
@@ -1,4 +1,3 @@
-include(${ClickHouse_SOURCE_DIR}/cmake/embed_binary.cmake)
set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/cctz")
set (SRCS
@@ -23,12 +22,10 @@ if (OS_FREEBSD)
endif ()
# Related to time_zones table:
-# StorageSystemTimeZones.generated.cpp is autogenerated each time during a build
-# data in this file will be used to populate the system.time_zones table, this is specific to OS_LINUX
-# as the library that's built using embedded tzdata is also specific to OS_LINUX
-set(SYSTEM_STORAGE_TZ_FILE "${PROJECT_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp")
+# TimeZones.generated.cpp is autogenerated each time during a build
+set(TIMEZONES_FILE "${CMAKE_CURRENT_BINARY_DIR}/TimeZones.generated.cpp")
# remove existing copies so that its generated fresh on each build.
-file(REMOVE ${SYSTEM_STORAGE_TZ_FILE})
+file(REMOVE ${TIMEZONES_FILE})
# get the list of timezones from tzdata shipped with cctz
set(TZDIR "${LIBRARY_DIR}/testdata/zoneinfo")
@@ -36,28 +33,44 @@ file(STRINGS "${LIBRARY_DIR}/testdata/version" TZDATA_VERSION)
set_property(GLOBAL PROPERTY TZDATA_VERSION_PROP "${TZDATA_VERSION}")
message(STATUS "Packaging with tzdata version: ${TZDATA_VERSION}")
-set(TIMEZONE_RESOURCE_FILES)
-
# each file in that dir (except of tab and localtime) store the info about timezone
execute_process(COMMAND
bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | LC_ALL=C sort | paste -sd ';' -"
OUTPUT_STRIP_TRAILING_WHITESPACE
OUTPUT_VARIABLE TIMEZONES)
-file(APPEND ${SYSTEM_STORAGE_TZ_FILE} "// autogenerated by ClickHouse/contrib/cctz-cmake/CMakeLists.txt\n")
-file(APPEND ${SYSTEM_STORAGE_TZ_FILE} "const char * auto_time_zones[] {\n" )
+file(APPEND ${TIMEZONES_FILE} "// autogenerated by ClickHouse/contrib/cctz-cmake/CMakeLists.txt\n")
+file(APPEND ${TIMEZONES_FILE} "#include \n")
+
+set (COUNTER 1)
+foreach(TIMEZONE ${TIMEZONES})
+ file(APPEND ${TIMEZONES_FILE} "INCBIN(resource_timezone${COUNTER}, \"${TZDIR}/${TIMEZONE}\");\n")
+ MATH(EXPR COUNTER "${COUNTER}+1")
+endforeach(TIMEZONE)
+
+file(APPEND ${TIMEZONES_FILE} "const char * auto_time_zones[] {\n" )
foreach(TIMEZONE ${TIMEZONES})
- file(APPEND ${SYSTEM_STORAGE_TZ_FILE} " \"${TIMEZONE}\",\n")
- list(APPEND TIMEZONE_RESOURCE_FILES "${TIMEZONE}")
+ file(APPEND ${TIMEZONES_FILE} " \"${TIMEZONE}\",\n")
+ MATH(EXPR COUNTER "${COUNTER}+1")
endforeach(TIMEZONE)
-file(APPEND ${SYSTEM_STORAGE_TZ_FILE} " nullptr};\n")
-clickhouse_embed_binaries(
- TARGET tzdata
- RESOURCE_DIR "${TZDIR}"
- RESOURCES ${TIMEZONE_RESOURCE_FILES}
-)
-add_dependencies(_cctz tzdata)
-target_link_libraries(_cctz INTERFACE "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}")
+
+file(APPEND ${TIMEZONES_FILE} " nullptr\n};\n\n")
+
+file(APPEND ${TIMEZONES_FILE} "#include \n\n")
+file(APPEND ${TIMEZONES_FILE} "std::string_view getTimeZone(const char * name)\n{\n" )
+
+set (COUNTER 1)
+foreach(TIMEZONE ${TIMEZONES})
+ file(APPEND ${TIMEZONES_FILE} " if (std::string_view(\"${TIMEZONE}\") == name) return { reinterpret_cast(gresource_timezone${COUNTER}Data), gresource_timezone${COUNTER}Size };\n")
+ MATH(EXPR COUNTER "${COUNTER}+1")
+endforeach(TIMEZONE)
+
+file(APPEND ${TIMEZONES_FILE} " return {};\n")
+file(APPEND ${TIMEZONES_FILE} "}\n")
+
+add_library (tzdata ${TIMEZONES_FILE})
+target_link_libraries(tzdata ch_contrib::incbin)
+target_link_libraries(_cctz tzdata)
add_library(ch_contrib::cctz ALIAS _cctz)
diff --git a/contrib/cityhash102/include/city.h b/contrib/cityhash102/include/city.h
index 77d4c988cdd..87363d16444 100644
--- a/contrib/cityhash102/include/city.h
+++ b/contrib/cityhash102/include/city.h
@@ -61,11 +61,24 @@ namespace CityHash_v1_0_2
typedef uint8_t uint8;
typedef uint32_t uint32;
typedef uint64_t uint64;
-typedef std::pair uint128;
+/// Represent an unsigned integer of 128 bits as it's used in CityHash.
+/// Originally CityHash used `std::pair` instead of this struct,
+/// however the members `first` and `second` could be easily confused so they were renamed to `low64` and `high64`:
+/// `first` -> `low64`, `second` -> `high64`.
+struct uint128
+{
+ uint64 low64 = 0;
+ uint64 high64 = 0;
-inline uint64 Uint128Low64(const uint128& x) { return x.first; }
-inline uint64 Uint128High64(const uint128& x) { return x.second; }
+ uint128() = default;
+ uint128(uint64 low64_, uint64 high64_) : low64(low64_), high64(high64_) {}
+ friend bool operator ==(const uint128 & x, const uint128 & y) { return (x.low64 == y.low64) && (x.high64 == y.high64); }
+ friend bool operator !=(const uint128 & x, const uint128 & y) { return !(x == y); }
+};
+
+inline uint64 Uint128Low64(const uint128 & x) { return x.low64; }
+inline uint64 Uint128High64(const uint128 & x) { return x.high64; }
// Hash function for a byte array.
uint64 CityHash64(const char *buf, size_t len);
diff --git a/contrib/hashidsxx b/contrib/hashidsxx
deleted file mode 160000
index 783f6911ccf..00000000000
--- a/contrib/hashidsxx
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 783f6911ccfdaca83e3cfac084c4aad888a80cee
diff --git a/contrib/hashidsxx-cmake/CMakeLists.txt b/contrib/hashidsxx-cmake/CMakeLists.txt
deleted file mode 100644
index 17f3888bd94..00000000000
--- a/contrib/hashidsxx-cmake/CMakeLists.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/hashidsxx")
-
-set (SRCS
- "${LIBRARY_DIR}/hashids.cpp"
-)
-
-set (HDRS
- "${LIBRARY_DIR}/hashids.h"
-)
-
-add_library(_hashidsxx ${SRCS} ${HDRS})
-target_include_directories(_hashidsxx SYSTEM PUBLIC "${LIBRARY_DIR}")
-
-add_library(ch_contrib::hashidsxx ALIAS _hashidsxx)
diff --git a/contrib/idxd-config b/contrib/idxd-config
index f6605c41a73..a836ce0e420 160000
--- a/contrib/idxd-config
+++ b/contrib/idxd-config
@@ -1 +1 @@
-Subproject commit f6605c41a735e3fdfef2d2d18655a33af6490b99
+Subproject commit a836ce0e42052a69bffbbc14239ab4097f3b77f1
diff --git a/contrib/incbin b/contrib/incbin
new file mode 160000
index 00000000000..6e576cae5ab
--- /dev/null
+++ b/contrib/incbin
@@ -0,0 +1 @@
+Subproject commit 6e576cae5ab5810f25e2631f2e0b80cbe7dc8cbf
diff --git a/contrib/incbin-cmake/CMakeLists.txt b/contrib/incbin-cmake/CMakeLists.txt
new file mode 100644
index 00000000000..5778cf83c22
--- /dev/null
+++ b/contrib/incbin-cmake/CMakeLists.txt
@@ -0,0 +1,8 @@
+set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/incbin")
+add_library(_incbin INTERFACE)
+target_include_directories(_incbin SYSTEM INTERFACE ${LIBRARY_DIR})
+add_library(ch_contrib::incbin ALIAS _incbin)
+
+# Warning "incbin is incompatible with bitcode. Using the library will break upload to App Store if you have bitcode enabled.
+# Add `#define INCBIN_SILENCE_BITCODE_WARNING` before including this header to silence this warning."
+target_compile_definitions(_incbin INTERFACE INCBIN_SILENCE_BITCODE_WARNING)
diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt
index 97f723bb540..15e965ed841 100644
--- a/contrib/jemalloc-cmake/CMakeLists.txt
+++ b/contrib/jemalloc-cmake/CMakeLists.txt
@@ -1,5 +1,5 @@
if (SANITIZE OR NOT (
- ((OS_LINUX OR OS_FREEBSD) AND (ARCH_AMD64 OR ARCH_AARCH64 OR ARCH_PPC64LE OR ARCH_RISCV64)) OR
+ ((OS_LINUX OR OS_FREEBSD) AND (ARCH_AMD64 OR ARCH_AARCH64 OR ARCH_PPC64LE OR ARCH_RISCV64 OR ARCH_S390X)) OR
(OS_DARWIN AND (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG"))
))
if (ENABLE_JEMALLOC)
@@ -17,17 +17,17 @@ if (NOT ENABLE_JEMALLOC)
endif ()
if (NOT OS_LINUX)
- message (WARNING "jemalloc support on non-linux is EXPERIMENTAL")
+ message (WARNING "jemalloc support on non-Linux is EXPERIMENTAL")
endif()
if (OS_LINUX)
- # ThreadPool select job randomly, and there can be some threads that had been
- # performed some memory heavy task before and will be inactive for some time,
- # but until it will became active again, the memory will not be freed since by
- # default each thread has it's own arena, but there should be not more then
+ # ThreadPool select job randomly, and there can be some threads that have been
+ # performed some memory-heavy tasks before and will be inactive for some time,
+ # but until it becomes active again, the memory will not be freed since, by
+ # default, each thread has its arena, but there should be no more than
# 4*CPU arenas (see opt.nareans description).
#
- # By enabling percpu_arena number of arenas limited to number of CPUs and hence
+ # By enabling percpu_arena number of arenas is limited to the number of CPUs, and hence
# this problem should go away.
#
# muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to
@@ -38,7 +38,7 @@ if (OS_LINUX)
else()
set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000")
endif()
-# CACHE variable is empty, to allow changing defaults without necessity
+# CACHE variable is empty to allow changing defaults without the necessity
# to purge cache
set (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE "" CACHE STRING "Change default configuration string of JEMalloc" )
if (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE)
@@ -148,6 +148,8 @@ elseif (ARCH_PPC64LE)
set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_ppc64le")
elseif (ARCH_RISCV64)
set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_riscv64")
+elseif (ARCH_S390X)
+ set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_s390x")
else ()
message (FATAL_ERROR "internal jemalloc: This arch is not supported")
endif ()
@@ -170,16 +172,13 @@ endif ()
target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_PROF=1)
-if (USE_UNWIND)
- # jemalloc provides support for two different libunwind flavors: the original HP libunwind and the one coming with gcc / g++ / libstdc++.
- # The latter is identified by `JEMALLOC_PROF_LIBGCC` and uses `_Unwind_Backtrace` method instead of `unw_backtrace`.
- # At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracing.
-
- # ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1).
-
- target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1)
- target_link_libraries (_jemalloc PRIVATE unwind)
-endif ()
+# jemalloc provides support for two different libunwind flavors: the original HP libunwind and the one coming with gcc / g++ / libstdc++.
+# The latter is identified by `JEMALLOC_PROF_LIBGCC` and uses `_Unwind_Backtrace` method instead of `unw_backtrace`.
+# At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracking.
+#
+# ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1).
+target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1)
+target_link_libraries (_jemalloc PRIVATE unwind)
# for RTLD_NEXT
target_compile_options(_jemalloc PRIVATE -D_GNU_SOURCE)
diff --git a/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in
new file mode 100644
index 00000000000..531f2bca0c2
--- /dev/null
+++ b/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -0,0 +1,435 @@
+/* include/jemalloc/internal/jemalloc_internal_defs.h. Generated from jemalloc_internal_defs.h.in by configure. */
+#ifndef JEMALLOC_INTERNAL_DEFS_H_
+#define JEMALLOC_INTERNAL_DEFS_H_
+/*
+ * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all
+ * public APIs to be prefixed. This makes it possible, with some care, to use
+ * multiple allocators simultaneously.
+ */
+/* #undef JEMALLOC_PREFIX */
+/* #undef JEMALLOC_CPREFIX */
+
+/*
+ * Define overrides for non-standard allocator-related functions if they are
+ * present on the system.
+ */
+#define JEMALLOC_OVERRIDE___LIBC_CALLOC
+#define JEMALLOC_OVERRIDE___LIBC_FREE
+#define JEMALLOC_OVERRIDE___LIBC_MALLOC
+#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN
+#define JEMALLOC_OVERRIDE___LIBC_REALLOC
+#define JEMALLOC_OVERRIDE___LIBC_VALLOC
+#define JEMALLOC_OVERRIDE___LIBC_PVALLOC
+/* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */
+
+/*
+ * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs.
+ * For shared libraries, symbol visibility mechanisms prevent these symbols
+ * from being exported, but for static libraries, naming collisions are a real
+ * possibility.
+ */
+#define JEMALLOC_PRIVATE_NAMESPACE je_
+
+/*
+ * Hyper-threaded CPUs may need a special instruction inside spin loops in
+ * order to yield to another virtual CPU.
+ */
+#define CPU_SPINWAIT
+/* 1 if CPU_SPINWAIT is defined, 0 otherwise. */
+#define HAVE_CPU_SPINWAIT 0
+
+/*
+ * Number of significant bits in virtual addresses. This may be less than the
+ * total number of bits in a pointer, e.g. on x64, for which the uppermost 16
+ * bits are the same as bit 47.
+ */
+#define LG_VADDR 64
+
+/* Defined if C11 atomics are available. */
+#define JEMALLOC_C11_ATOMICS
+
+/* Defined if GCC __atomic atomics are available. */
+#define JEMALLOC_GCC_ATOMIC_ATOMICS
+/* and the 8-bit variant support. */
+#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS
+
+/* Defined if GCC __sync atomics are available. */
+#define JEMALLOC_GCC_SYNC_ATOMICS
+/* and the 8-bit variant support. */
+#define JEMALLOC_GCC_U8_SYNC_ATOMICS
+
+/*
+ * Defined if __builtin_clz() and __builtin_clzl() are available.
+ */
+#define JEMALLOC_HAVE_BUILTIN_CLZ
+
+/*
+ * Defined if os_unfair_lock_*() functions are available, as provided by Darwin.
+ */
+/* #undef JEMALLOC_OS_UNFAIR_LOCK */
+
+/* Defined if syscall(2) is usable. */
+#define JEMALLOC_USE_SYSCALL
+
+/*
+ * Defined if secure_getenv(3) is available.
+ */
+#define JEMALLOC_HAVE_SECURE_GETENV
+
+/*
+ * Defined if issetugid(2) is available.
+ */
+/* #undef JEMALLOC_HAVE_ISSETUGID */
+
+/* Defined if pthread_atfork(3) is available. */
+#define JEMALLOC_HAVE_PTHREAD_ATFORK
+
+/* Defined if pthread_setname_np(3) is available. */
+#define JEMALLOC_HAVE_PTHREAD_SETNAME_NP
+
+/* Defined if pthread_getname_np(3) is available. */
+#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP
+
+/* Defined if pthread_get_name_np(3) is available. */
+/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */
+
+/*
+ * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
+ */
+#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
+
+/*
+ * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.
+ */
+#define JEMALLOC_HAVE_CLOCK_MONOTONIC
+
+/*
+ * Defined if mach_absolute_time() is available.
+ */
+/* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */
+
+/*
+ * Defined if clock_gettime(CLOCK_REALTIME, ...) is available.
+ */
+#define JEMALLOC_HAVE_CLOCK_REALTIME
+
+/*
+ * Defined if _malloc_thread_cleanup() exists. At least in the case of
+ * FreeBSD, pthread_key_create() allocates, which if used during malloc
+ * bootstrapping will cause recursion into the pthreads library. Therefore, if
+ * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in
+ * malloc_tsd.
+ */
+/* #undef JEMALLOC_MALLOC_THREAD_CLEANUP */
+
+/*
+ * Defined if threaded initialization is known to be safe on this platform.
+ * Among other things, it must be possible to initialize a mutex without
+ * triggering allocation in order for threaded allocation to be safe.
+ */
+#define JEMALLOC_THREADED_INIT
+
+/*
+ * Defined if the pthreads implementation defines
+ * _pthread_mutex_init_calloc_cb(), in which case the function is used in order
+ * to avoid recursive allocation during mutex initialization.
+ */
+/* #undef JEMALLOC_MUTEX_INIT_CB */
+
+/* Non-empty if the tls_model attribute is supported. */
+#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
+
+/*
+ * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
+ * inline functions.
+ */
+/* #undef JEMALLOC_DEBUG */
+
+/* JEMALLOC_STATS enables statistics calculation. */
+#define JEMALLOC_STATS
+
+/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */
+/* #undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API */
+
+/* JEMALLOC_PROF enables allocation profiling. */
+/* #undef JEMALLOC_PROF */
+
+/* Use libunwind for profile backtracing if defined. */
+/* #undef JEMALLOC_PROF_LIBUNWIND */
+
+/* Use libgcc for profile backtracing if defined. */
+/* #undef JEMALLOC_PROF_LIBGCC */
+
+/* Use gcc intrinsics for profile backtracing if defined. */
+/* #undef JEMALLOC_PROF_GCC */
+
+/* JEMALLOC_PAGEID enabled page id */
+/* #undef JEMALLOC_PAGEID */
+
+/* JEMALLOC_HAVE_PRCTL checks prctl */
+#define JEMALLOC_HAVE_PRCTL
+
+/*
+ * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage
+ * segment (DSS).
+ */
+#define JEMALLOC_DSS
+
+/* Support memory filling (junk/zero). */
+#define JEMALLOC_FILL
+
+/* Support utrace(2)-based tracing. */
+/* #undef JEMALLOC_UTRACE */
+
+/* Support utrace(2)-based tracing (label based signature). */
+/* #undef JEMALLOC_UTRACE_LABEL */
+
+/* Support optional abort() on OOM. */
+/* #undef JEMALLOC_XMALLOC */
+
+/* Support lazy locking (avoid locking unless a second thread is launched). */
+/* #undef JEMALLOC_LAZY_LOCK */
+
+/*
+ * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size
+ * classes).
+ */
+/* #undef LG_QUANTUM */
+
+/* One page is 2^LG_PAGE bytes. */
+#define LG_PAGE 12
+
+/* Maximum number of regions in a slab. */
+/* #undef CONFIG_LG_SLAB_MAXREGS */
+
+/*
+ * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the
+ * system does not explicitly support huge pages; system calls that require
+ * explicit huge page support are separately configured.
+ */
+#define LG_HUGEPAGE 20
+
+/*
+ * If defined, adjacent virtual memory mappings with identical attributes
+ * automatically coalesce, and they fragment when changes are made to subranges.
+ * This is the normal order of things for mmap()/munmap(), but on Windows
+ * VirtualAlloc()/VirtualFree() operations must be precisely matched, i.e.
+ * mappings do *not* coalesce/fragment.
+ */
+#define JEMALLOC_MAPS_COALESCE
+
+/*
+ * If defined, retain memory for later reuse by default rather than using e.g.
+ * munmap() to unmap freed extents. This is enabled on 64-bit Linux because
+ * common sequences of mmap()/munmap() calls will cause virtual memory map
+ * holes.
+ */
+#define JEMALLOC_RETAIN
+
+/* TLS is used to map arenas and magazine caches to threads. */
+#define JEMALLOC_TLS
+
+/*
+ * Used to mark unreachable code to quiet "end of non-void" compiler warnings.
+ * Don't use this directly; instead use unreachable() from util.h
+ */
+#define JEMALLOC_INTERNAL_UNREACHABLE __builtin_unreachable
+
+/*
+ * ffs*() functions to use for bitmapping. Don't use these directly; instead,
+ * use ffs_*() from util.h.
+ */
+#define JEMALLOC_INTERNAL_FFSLL __builtin_ffsll
+#define JEMALLOC_INTERNAL_FFSL __builtin_ffsl
+#define JEMALLOC_INTERNAL_FFS __builtin_ffs
+
+/*
+ * popcount*() functions to use for bitmapping.
+ */
+#define JEMALLOC_INTERNAL_POPCOUNTL __builtin_popcountl
+#define JEMALLOC_INTERNAL_POPCOUNT __builtin_popcount
+
+/*
+ * If defined, explicitly attempt to more uniformly distribute large allocation
+ * pointer alignments across all cache indices.
+ */
+#define JEMALLOC_CACHE_OBLIVIOUS
+
+/*
+ * If defined, enable logging facilities. We make this a configure option to
+ * avoid taking extra branches everywhere.
+ */
+/* #undef JEMALLOC_LOG */
+
+/*
+ * If defined, use readlinkat() (instead of readlink()) to follow
+ * /etc/malloc_conf.
+ */
+/* #undef JEMALLOC_READLINKAT */
+
+/*
+ * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
+ */
+/* #undef JEMALLOC_ZONE */
+
+/*
+ * Methods for determining whether the OS overcommits.
+ * JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's
+ * /proc/sys/vm.overcommit_memory file.
+ * JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl.
+ */
+/* #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT */
+#define JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
+
+/* Defined if madvise(2) is available. */
+#define JEMALLOC_HAVE_MADVISE
+
+/*
+ * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE
+ * arguments to madvise(2).
+ */
+#define JEMALLOC_HAVE_MADVISE_HUGE
+
+/*
+ * Methods for purging unused pages differ between operating systems.
+ *
+ * madvise(..., MADV_FREE) : This marks pages as being unused, such that they
+ * will be discarded rather than swapped out.
+ * madvise(..., MADV_DONTNEED) : If JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS is
+ * defined, this immediately discards pages,
+ * such that new pages will be demand-zeroed if
+ * the address region is later touched;
+ * otherwise this behaves similarly to
+ * MADV_FREE, though typically with higher
+ * system overhead.
+ */
+#define JEMALLOC_PURGE_MADVISE_FREE
+#define JEMALLOC_PURGE_MADVISE_DONTNEED
+#define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
+
+/* Defined if madvise(2) is available but MADV_FREE is not (x86 Linux only). */
+/* #undef JEMALLOC_DEFINE_MADVISE_FREE */
+
+/*
+ * Defined if MADV_DO[NT]DUMP is supported as an argument to madvise.
+ */
+#define JEMALLOC_MADVISE_DONTDUMP
+
+/*
+ * Defined if MADV_[NO]CORE is supported as an argument to madvise.
+ */
+/* #undef JEMALLOC_MADVISE_NOCORE */
+
+/* Defined if mprotect(2) is available. */
+#define JEMALLOC_HAVE_MPROTECT
+
+/*
+ * Defined if transparent huge pages (THPs) are supported via the
+ * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled.
+ */
+/* #undef JEMALLOC_THP */
+
+/* Defined if posix_madvise is available. */
+/* #undef JEMALLOC_HAVE_POSIX_MADVISE */
+
+/*
+ * Method for purging unused pages using posix_madvise.
+ *
+ * posix_madvise(..., POSIX_MADV_DONTNEED)
+ */
+/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */
+/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */
+
+/*
+ * Defined if memcntl page admin call is supported
+ */
+/* #undef JEMALLOC_HAVE_MEMCNTL */
+
+/*
+ * Defined if malloc_size is supported
+ */
+/* #undef JEMALLOC_HAVE_MALLOC_SIZE */
+
+/* Define if operating system has alloca.h header. */
+#define JEMALLOC_HAS_ALLOCA_H
+
+/* C99 restrict keyword supported. */
+#define JEMALLOC_HAS_RESTRICT
+
+/* For use by hash code. */
+#define JEMALLOC_BIG_ENDIAN
+
+/* sizeof(int) == 2^LG_SIZEOF_INT. */
+#define LG_SIZEOF_INT 2
+
+/* sizeof(long) == 2^LG_SIZEOF_LONG. */
+#define LG_SIZEOF_LONG 3
+
+/* sizeof(long long) == 2^LG_SIZEOF_LONG_LONG. */
+#define LG_SIZEOF_LONG_LONG 3
+
+/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */
+#define LG_SIZEOF_INTMAX_T 3
+
+/* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */
+/* #undef JEMALLOC_GLIBC_MALLOC_HOOK */
+
+/* glibc memalign hook. */
+/* #undef JEMALLOC_GLIBC_MEMALIGN_HOOK */
+
+/* pthread support */
+#define JEMALLOC_HAVE_PTHREAD
+
+/* dlsym() support */
+#define JEMALLOC_HAVE_DLSYM
+
+/* Adaptive mutex support in pthreads. */
+#define JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP
+
+/* GNU specific sched_getcpu support */
+#define JEMALLOC_HAVE_SCHED_GETCPU
+
+/* GNU specific sched_setaffinity support */
+#define JEMALLOC_HAVE_SCHED_SETAFFINITY
+
+/*
+ * If defined, all the features necessary for background threads are present.
+ */
+#define JEMALLOC_BACKGROUND_THREAD
+
+/*
+ * If defined, jemalloc symbols are not exported (doesn't work when
+ * JEMALLOC_PREFIX is not defined).
+ */
+/* #undef JEMALLOC_EXPORT */
+
+/* config.malloc_conf options string. */
+#define JEMALLOC_CONFIG_MALLOC_CONF ""
+
+/* If defined, jemalloc takes the malloc/free/etc. symbol names. */
+#define JEMALLOC_IS_MALLOC
+
+/*
+ * Defined if strerror_r returns char * if _GNU_SOURCE is defined.
+ */
+#define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE
+
+/* Performs additional safety checks when defined. */
+/* #undef JEMALLOC_OPT_SAFETY_CHECKS */
+
+/* Is C++ support being built? */
+#define JEMALLOC_ENABLE_CXX
+
+/* Performs additional size checks when defined. */
+/* #undef JEMALLOC_OPT_SIZE_CHECKS */
+
+/* Allows sampled junk and stash for checking use-after-free when defined. */
+/* #undef JEMALLOC_UAF_DETECTION */
+
+/* Darwin VM_MAKE_TAG support */
+/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */
+
+/* If defined, realloc(ptr, 0) defaults to "free" instead of "alloc". */
+#define JEMALLOC_ZERO_REALLOC_DEFAULT_FREE
+
+#endif /* JEMALLOC_INTERNAL_DEFS_H_ */
diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt
index a13e4f0f60a..b7e59e2c9a3 100644
--- a/contrib/libcxx-cmake/CMakeLists.txt
+++ b/contrib/libcxx-cmake/CMakeLists.txt
@@ -61,9 +61,7 @@ target_include_directories(cxx SYSTEM BEFORE PUBLIC $<$:$
target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI)
# Enable capturing stack traces for all exceptions.
-if (USE_UNWIND)
- target_compile_definitions(cxx PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1)
-endif ()
+target_compile_definitions(cxx PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1)
if (USE_MUSL)
target_compile_definitions(cxx PUBLIC -D_LIBCPP_HAS_MUSL_LIBC=1)
diff --git a/contrib/libcxxabi-cmake/CMakeLists.txt b/contrib/libcxxabi-cmake/CMakeLists.txt
index 0473527912e..c7ee34e6e28 100644
--- a/contrib/libcxxabi-cmake/CMakeLists.txt
+++ b/contrib/libcxxabi-cmake/CMakeLists.txt
@@ -35,12 +35,10 @@ target_include_directories(cxxabi SYSTEM BEFORE
)
target_compile_definitions(cxxabi PRIVATE -D_LIBCPP_BUILDING_LIBRARY)
target_compile_options(cxxabi PRIVATE -nostdinc++ -fno-sanitize=undefined -Wno-macro-redefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast.
-target_link_libraries(cxxabi PUBLIC ${EXCEPTION_HANDLING_LIBRARY})
+target_link_libraries(cxxabi PUBLIC unwind)
# Enable capturing stack traces for all exceptions.
-if (USE_UNWIND)
- target_compile_definitions(cxxabi PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1)
-endif ()
+target_compile_definitions(cxxabi PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1)
install(
TARGETS cxxabi
diff --git a/contrib/libhdfs3 b/contrib/libhdfs3
index 164b89253fa..377220ef351 160000
--- a/contrib/libhdfs3
+++ b/contrib/libhdfs3
@@ -1 +1 @@
-Subproject commit 164b89253fad7991bce77882f01b51ab81d19f3d
+Subproject commit 377220ef351ae24994a5fcd2b5fa3930d00c4db0
diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt
index e2f122e282a..a630a8e45c4 100644
--- a/contrib/libhdfs3-cmake/CMakeLists.txt
+++ b/contrib/libhdfs3-cmake/CMakeLists.txt
@@ -1,11 +1,11 @@
-if(NOT ARCH_AARCH64 AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE AND NOT ARCH_S390X)
+if(NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE AND NOT ARCH_S390X)
option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES})
elseif(ENABLE_HDFS)
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration")
endif()
if(NOT ENABLE_HDFS)
- message(STATUS "Not using hdfs")
+ message(STATUS "Not using HDFS")
return()
endif()
diff --git a/contrib/nlp-data-cmake/CMakeLists.txt b/contrib/nlp-data-cmake/CMakeLists.txt
deleted file mode 100644
index 5380269c479..00000000000
--- a/contrib/nlp-data-cmake/CMakeLists.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-include(${ClickHouse_SOURCE_DIR}/cmake/embed_binary.cmake)
-
-set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/nlp-data")
-
-add_library (_nlp_data INTERFACE)
-
-clickhouse_embed_binaries(
- TARGET nlp_dictionaries
- RESOURCE_DIR "${LIBRARY_DIR}"
- RESOURCES charset.zst tonality_ru.zst programming.zst
-)
-
-add_dependencies(_nlp_data nlp_dictionaries)
-target_link_libraries(_nlp_data INTERFACE "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}")
-add_library(ch_contrib::nlp_data ALIAS _nlp_data)
diff --git a/contrib/orc b/contrib/orc
index c5d7755ba0b..568d1d60c25 160000
--- a/contrib/orc
+++ b/contrib/orc
@@ -1 +1 @@
-Subproject commit c5d7755ba0b9a95631c8daea4d094101f26ec761
+Subproject commit 568d1d60c250af1890f226c182bc15bd8cc94cf1
diff --git a/contrib/qpl b/contrib/qpl
index 3f8f5cea277..faaf1935045 160000
--- a/contrib/qpl
+++ b/contrib/qpl
@@ -1 +1 @@
-Subproject commit 3f8f5cea27739f5261e8fd577dc233ffe88bf679
+Subproject commit faaf19350459c076e66bb5df11743c3fade59b73
diff --git a/contrib/re2 b/contrib/re2
index 13ebb377c6a..03da4fc0857 160000
--- a/contrib/re2
+++ b/contrib/re2
@@ -1 +1 @@
-Subproject commit 13ebb377c6ad763ca61d12dd6f88b1126bd0b911
+Subproject commit 03da4fc0857c285e3a26782f6bc8931c4c950df4
diff --git a/contrib/re2-cmake/CMakeLists.txt b/contrib/re2-cmake/CMakeLists.txt
index 19939c11ebf..305c2400c77 100644
--- a/contrib/re2-cmake/CMakeLists.txt
+++ b/contrib/re2-cmake/CMakeLists.txt
@@ -12,6 +12,7 @@ endif()
set(SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/re2")
set(RE2_SOURCES
+ ${SRC_DIR}/re2/bitmap256.cc
${SRC_DIR}/re2/bitstate.cc
${SRC_DIR}/re2/compile.cc
${SRC_DIR}/re2/dfa.cc
@@ -28,15 +29,16 @@ set(RE2_SOURCES
${SRC_DIR}/re2/regexp.cc
${SRC_DIR}/re2/set.cc
${SRC_DIR}/re2/simplify.cc
- ${SRC_DIR}/re2/stringpiece.cc
${SRC_DIR}/re2/tostring.cc
${SRC_DIR}/re2/unicode_casefold.cc
${SRC_DIR}/re2/unicode_groups.cc
+ ${SRC_DIR}/util/pcre.cc
${SRC_DIR}/util/rune.cc
${SRC_DIR}/util/strutil.cc
)
add_library(re2 ${RE2_SOURCES})
target_include_directories(re2 PUBLIC "${SRC_DIR}")
+target_link_libraries(re2 ch_contrib::abseil_str_format)
# Building re2 which is thread-safe and re2_st which is not.
# re2 changes its state during matching of regular expression, e.g. creates temporary DFA.
@@ -48,6 +50,7 @@ target_compile_definitions (re2_st PRIVATE NDEBUG NO_THREADS re2=re2_st)
target_include_directories (re2_st PRIVATE .)
target_include_directories (re2_st SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
target_include_directories (re2_st SYSTEM BEFORE PUBLIC ${SRC_DIR})
+target_link_libraries (re2_st ch_contrib::abseil_str_format)
file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/re2_st)
foreach (FILENAME filtered_re2.h re2.h set.h stringpiece.h)
@@ -60,17 +63,6 @@ foreach (FILENAME filtered_re2.h re2.h set.h stringpiece.h)
add_dependencies (re2_st transform_${FILENAME})
endforeach ()
-file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/util)
-foreach (FILENAME mutex.h)
- add_custom_command (OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}"
- COMMAND ${CMAKE_COMMAND} -DSOURCE_FILENAME="${SRC_DIR}/util/${FILENAME}"
- -DTARGET_FILENAME="${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}"
- -P "${CMAKE_CURRENT_SOURCE_DIR}/re2_transform.cmake"
- COMMENT "Creating ${FILENAME} for re2_st library.")
- add_custom_target (transform_${FILENAME} DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}")
- add_dependencies (re2_st transform_${FILENAME})
-endforeach ()
-
# NOTE: you should not change name of library here, since it is used to generate required header (see above)
add_library(ch_contrib::re2 ALIAS re2)
add_library(ch_contrib::re2_st ALIAS re2_st)
diff --git a/docker/images.json b/docker/images.json
index b4f3e755bd1..e8fc329a640 100644
--- a/docker/images.json
+++ b/docker/images.json
@@ -120,11 +120,12 @@
"docker/test/base": {
"name": "clickhouse/test-base",
"dependent": [
- "docker/test/stateless",
- "docker/test/integration/base",
"docker/test/fuzzer",
+ "docker/test/integration/base",
"docker/test/keeper-jepsen",
- "docker/test/server-jepsen"
+ "docker/test/server-jepsen",
+ "docker/test/sqllogic",
+ "docker/test/stateless"
]
},
"docker/test/integration/kerberized_hadoop": {
diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile
index 44967af4b32..8a6324aef88 100644
--- a/docker/keeper/Dockerfile
+++ b/docker/keeper/Dockerfile
@@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
esac
ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release"
-ARG VERSION="23.5.2.7"
+ARG VERSION="23.6.2.18"
ARG PACKAGES="clickhouse-keeper"
# user/group precreated explicitly with fixed uid/gid on purpose.
diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index dd21c8552d3..99e748c41d4 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -49,8 +49,8 @@ ENV CARGO_HOME=/rust/cargo
ENV PATH="/rust/cargo/bin:${PATH}"
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
chmod 777 -R /rust && \
- rustup toolchain install nightly && \
- rustup default nightly && \
+ rustup toolchain install nightly-2023-07-04 && \
+ rustup default nightly-2023-07-04 && \
rustup component add rust-src && \
rustup target add aarch64-unknown-linux-gnu && \
rustup target add x86_64-apple-darwin && \
@@ -58,6 +58,33 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
rustup target add aarch64-apple-darwin && \
rustup target add powerpc64le-unknown-linux-gnu
+# Create vendor cache for cargo.
+#
+# Note, that the config.toml for the root is used, you will not be able to
+# install any other crates, except those which had been vendored (since if
+# there is "replace-with" for some source, then cargo will not look to other
+# remotes except this).
+#
+# Notes for the command itself:
+# - --chown is required to preserve the rights
+# - unstable-options for -C
+# - chmod is required to fix the permissions, since builds are running from a different user
+# - copy of the Cargo.lock is required for proper dependencies versions
+# - cargo vendor --sync is requried to overcome [1] bug.
+#
+# [1]: https://github.com/rust-lang/wg-cargo-std-aware/issues/23
+COPY --chown=root:root /rust /rust/packages
+RUN cargo -Z unstable-options -C /rust/packages vendor > $CARGO_HOME/config.toml && \
+ cp "$(rustc --print=sysroot)"/lib/rustlib/src/rust/Cargo.lock "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/ && \
+ cargo -Z unstable-options -C /rust/packages vendor --sync "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/Cargo.toml && \
+ rm "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/Cargo.lock && \
+ sed -i "s#\"vendor\"#\"/rust/vendor\"#" $CARGO_HOME/config.toml && \
+ cat $CARGO_HOME/config.toml && \
+ mv /rust/packages/vendor /rust/vendor && \
+ chmod -R o=r+X /rust/vendor && \
+ ls -R -l /rust/packages && \
+ rm -r /rust/packages
+
# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
# A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work):
RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
@@ -89,7 +116,7 @@ RUN arch=${TARGETARCH:-amd64} \
&& dpkg -i /tmp/nfpm.deb \
&& rm /tmp/nfpm.deb
-ARG GO_VERSION=1.19.5
+ARG GO_VERSION=1.19.10
# We need go for clickhouse-diagnostics
RUN arch=${TARGETARCH:-amd64} \
&& curl -Lo /tmp/go.tgz "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" \
diff --git a/docker/packager/binary/rust b/docker/packager/binary/rust
new file mode 120000
index 00000000000..742dc49e9ac
--- /dev/null
+++ b/docker/packager/binary/rust
@@ -0,0 +1 @@
+../../../rust
\ No newline at end of file
diff --git a/docker/packager/packager b/docker/packager/packager
index 1b3df858cd2..e12bd55dde3 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -138,6 +138,7 @@ def parse_env_variables(
ARM_V80COMPAT_SUFFIX = "-aarch64-v80compat"
FREEBSD_SUFFIX = "-freebsd"
PPC_SUFFIX = "-ppc64le"
+ RISCV_SUFFIX = "-riscv64"
AMD64_COMPAT_SUFFIX = "-amd64-compat"
result = []
@@ -150,6 +151,7 @@ def parse_env_variables(
is_cross_arm = compiler.endswith(ARM_SUFFIX)
is_cross_arm_v80compat = compiler.endswith(ARM_V80COMPAT_SUFFIX)
is_cross_ppc = compiler.endswith(PPC_SUFFIX)
+ is_cross_riscv = compiler.endswith(RISCV_SUFFIX)
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
is_amd64_compat = compiler.endswith(AMD64_COMPAT_SUFFIX)
@@ -206,6 +208,11 @@ def parse_env_variables(
cmake_flags.append(
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake"
)
+ elif is_cross_riscv:
+ cc = compiler[: -len(RISCV_SUFFIX)]
+ cmake_flags.append(
+ "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-riscv64.cmake"
+ )
elif is_amd64_compat:
cc = compiler[: -len(AMD64_COMPAT_SUFFIX)]
result.append("DEB_ARCH=amd64")
@@ -370,6 +377,7 @@ def parse_args() -> argparse.Namespace:
"clang-16-aarch64",
"clang-16-aarch64-v80compat",
"clang-16-ppc64le",
+ "clang-16-riscv64",
"clang-16-amd64-compat",
"clang-16-freebsd",
),
diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine
index 8ab9bf7b077..7f453627601 100644
--- a/docker/server/Dockerfile.alpine
+++ b/docker/server/Dockerfile.alpine
@@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="23.5.2.7"
+ARG VERSION="23.6.2.18"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# user/group precreated explicitly with fixed uid/gid on purpose.
diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu
index b3b0cfe1510..1fa7b83ae16 100644
--- a/docker/server/Dockerfile.ubuntu
+++ b/docker/server/Dockerfile.ubuntu
@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:20.04
# see https://github.com/moby/moby/issues/4032#issuecomment-192327844
ARG DEBIAN_FRONTEND=noninteractive
@@ -11,18 +11,19 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
&& apt-get update \
&& apt-get upgrade -yq \
&& apt-get install --yes --no-install-recommends \
- apt-transport-https \
ca-certificates \
- dirmngr \
- gnupg2 \
- wget \
locales \
tzdata \
- && apt-get clean
+ wget \
+ && apt-get clean \
+ && rm -rf \
+ /var/lib/apt/lists/* \
+ /var/cache/debconf \
+ /tmp/*
ARG REPO_CHANNEL="stable"
-ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="23.5.2.7"
+ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
+ARG VERSION="23.6.2.18"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# set non-empty deb_location_url url to create a docker image
@@ -43,49 +44,68 @@ ARG single_binary_location_url=""
ARG TARGETARCH
-RUN arch=${TARGETARCH:-amd64} \
+# install from a web location with deb packages
+RUN arch="${TARGETARCH:-amd64}" \
&& if [ -n "${deb_location_url}" ]; then \
echo "installing from custom url with deb packages: ${deb_location_url}" \
- rm -rf /tmp/clickhouse_debs \
+ && rm -rf /tmp/clickhouse_debs \
&& mkdir -p /tmp/clickhouse_debs \
&& for package in ${PACKAGES}; do \
{ wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_${arch}.deb" -P /tmp/clickhouse_debs || \
wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_all.deb" -P /tmp/clickhouse_debs ; } \
|| exit 1 \
; done \
- && dpkg -i /tmp/clickhouse_debs/*.deb ; \
- elif [ -n "${single_binary_location_url}" ]; then \
+ && dpkg -i /tmp/clickhouse_debs/*.deb \
+ && rm -rf /tmp/* ; \
+ fi
+
+# install from a single binary
+RUN if [ -n "${single_binary_location_url}" ]; then \
echo "installing from single binary url: ${single_binary_location_url}" \
&& rm -rf /tmp/clickhouse_binary \
&& mkdir -p /tmp/clickhouse_binary \
&& wget --progress=bar:force:noscroll "${single_binary_location_url}" -O /tmp/clickhouse_binary/clickhouse \
&& chmod +x /tmp/clickhouse_binary/clickhouse \
- && /tmp/clickhouse_binary/clickhouse install --user "clickhouse" --group "clickhouse" ; \
- else \
- mkdir -p /etc/apt/sources.list.d \
- && apt-key adv --keyserver keyserver.ubuntu.com --recv 8919F6BD2B48D754 \
- && echo ${REPOSITORY} > /etc/apt/sources.list.d/clickhouse.list \
+ && /tmp/clickhouse_binary/clickhouse install --user "clickhouse" --group "clickhouse" \
+ && rm -rf /tmp/* ; \
+ fi
+
+# A fallback to installation from ClickHouse repository
+RUN if ! clickhouse local -q "SELECT ''" > /dev/null 2>&1; then \
+ apt-get update \
+ && apt-get install --yes --no-install-recommends \
+ apt-transport-https \
+ ca-certificates \
+ dirmngr \
+ gnupg2 \
+ && mkdir -p /etc/apt/sources.list.d \
+ && GNUPGHOME=$(mktemp -d) \
+ && GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring \
+ --keyring /usr/share/keyrings/clickhouse-keyring.gpg \
+ --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 8919F6BD2B48D754 \
+ && rm -r "$GNUPGHOME" \
+ && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
+ && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
&& echo "installing from repository: ${REPOSITORY}" \
&& apt-get update \
- && apt-get --yes -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" upgrade \
&& for package in ${PACKAGES}; do \
packages="${packages} ${package}=${VERSION}" \
; done \
&& apt-get install --allow-unauthenticated --yes --no-install-recommends ${packages} || exit 1 \
- ; fi \
- && clickhouse-local -q 'SELECT * FROM system.build_options' \
- && rm -rf \
- /var/lib/apt/lists/* \
- /var/cache/debconf \
- /tmp/* \
- && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client \
- && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client
-
-RUN apt-get autoremove --purge -yq libksba8 && \
- apt-get autoremove -yq
+ && rm -rf \
+ /var/lib/apt/lists/* \
+ /var/cache/debconf \
+ /tmp/* \
+ && apt-get autoremove --purge -yq libksba8 \
+ && apt-get autoremove -yq \
+ ; fi
+# post install
# we need to allow "others" access to clickhouse folder, because docker container
# can be started with arbitrary uid (openshift usecase)
+RUN clickhouse-local -q 'SELECT * FROM system.build_options' \
+ && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client \
+ && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
diff --git a/docker/server/README.md b/docker/server/README.md
index 18dce492123..6200acbd30c 100644
--- a/docker/server/README.md
+++ b/docker/server/README.md
@@ -20,7 +20,6 @@ For more information and documentation see https://clickhouse.com/.
- The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.
- The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A). Most ARM CPUs after 2017 support ARMv8.2-A. A notable exception is Raspberry Pi 4 from 2019 whose CPU only supports ARMv8.0-A.
-- Since the Clickhouse 23.3 Ubuntu image started using `ubuntu:22.04` as its base image, it requires docker version >= `20.10.10`, or use `docker run -- privileged` instead. Alternatively, try the Clickhouse Alpine image.
## How to use this image
@@ -98,8 +97,8 @@ docker run -d \
You may also want to mount:
-* `/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustmenets
-* `/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustmenets
+* `/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustments
+* `/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustments
* `/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below).
### Linux capabilities
diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile
index ffb13fc774d..da4baa8c687 100644
--- a/docker/test/fasttest/Dockerfile
+++ b/docker/test/fasttest/Dockerfile
@@ -9,6 +9,7 @@ RUN apt-get update \
expect \
file \
lsof \
+ odbcinst \
psmisc \
python3 \
python3-lxml \
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index dab873377ce..60e6199aaa4 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -80,7 +80,7 @@ function start_server
function clone_root
{
- git config --global --add safe.directory "$FASTTEST_SOURCE"
+ [ "$UID" -eq 0 ] && git config --global --add safe.directory "$FASTTEST_SOURCE"
git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt"
(
@@ -141,17 +141,17 @@ function clone_submodules
contrib/jemalloc
contrib/replxx
contrib/wyhash
- contrib/hashidsxx
contrib/c-ares
contrib/morton-nd
contrib/xxHash
contrib/simdjson
contrib/liburing
contrib/libfiu
+ contrib/incbin
)
git submodule sync
- git submodule update --jobs=16 --depth 1 --init "${SUBMODULES_TO_UPDATE[@]}"
+ git submodule update --jobs=16 --depth 1 --single-branch --init "${SUBMODULES_TO_UPDATE[@]}"
git submodule foreach git reset --hard
git submodule foreach git checkout @ -f
git submodule foreach git clean -xfd
@@ -166,7 +166,6 @@ function run_cmake
"-DENABLE_UTILS=0"
"-DENABLE_EMBEDDED_COMPILER=0"
"-DENABLE_THINLTO=0"
- "-DUSE_UNWIND=1"
"-DENABLE_NURAFT=1"
"-DENABLE_SIMDJSON=1"
"-DENABLE_JEMALLOC=1"
@@ -202,10 +201,11 @@ function build
| ts '%Y-%m-%d %H:%M:%S' \
| tee "$FASTTEST_OUTPUT/test_result.txt"
if [ "$COPY_CLICKHOUSE_BINARY_TO_OUTPUT" -eq "1" ]; then
- cp programs/clickhouse "$FASTTEST_OUTPUT/clickhouse"
+ mkdir -p "$FASTTEST_OUTPUT/binaries/"
+ cp programs/clickhouse "$FASTTEST_OUTPUT/binaries/clickhouse"
- strip programs/clickhouse -o "$FASTTEST_OUTPUT/clickhouse-stripped"
- zstd --threads=0 "$FASTTEST_OUTPUT/clickhouse-stripped"
+ strip programs/clickhouse -o programs/clickhouse-stripped
+ zstd --threads=0 programs/clickhouse-stripped -o "$FASTTEST_OUTPUT/binaries/clickhouse-stripped.zst"
fi
ccache_status
ccache --evict-older-than 1d ||:
diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index d2c8de7a211..5cda0831a84 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -291,7 +291,7 @@ quit
if [ "$server_died" == 1 ]
then
# The server has died.
- if ! rg --text -o 'Received signal.*|Logical error.*|Assertion.*failed|Failed assertion.*|.*runtime error: .*|.*is located.*|(SUMMARY|ERROR): [a-zA-Z]+Sanitizer:.*|.*_LIBCPP_ASSERT.*' server.log > description.txt
+ if ! rg --text -o 'Received signal.*|Logical error.*|Assertion.*failed|Failed assertion.*|.*runtime error: .*|.*is located.*|(SUMMARY|ERROR): [a-zA-Z]+Sanitizer:.*|.*_LIBCPP_ASSERT.*|.*Child process was terminated by signal 9.*' server.log > description.txt
then
echo "Lost connection to server. See the logs." > description.txt
fi
diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile
index de8efa20af4..270b40e23a6 100644
--- a/docker/test/integration/base/Dockerfile
+++ b/docker/test/integration/base/Dockerfile
@@ -46,12 +46,13 @@ RUN arch=${TARGETARCH:-amd64} \
arm64) rarch=aarch64 ;; \
esac \
&& cd /tmp \
- && curl -o mysql-odbc.rpm "https://cdn.mysql.com/archives/mysql-connector-odbc-8.0/mysql-connector-odbc-8.0.27-1.el8.${rarch}.rpm" \
+ && curl -o mysql-odbc.rpm "https://cdn.mysql.com/archives/mysql-connector-odbc-8.0/mysql-connector-odbc-8.0.32-1.el9.${rarch}.rpm" \
&& rpm2archive mysql-odbc.rpm \
&& tar xf mysql-odbc.rpm.tgz -C / ./usr/lib64/ \
- && LINK_DIR=$(dpkg -L libodbc1 | rg '^/usr/lib/.*-linux-gnu/odbc$') \
- && ln -s /usr/lib64/libmyodbc8a.so "$LINK_DIR" \
- && ln -s /usr/lib64/libmyodbc8a.so "$LINK_DIR"/libmyodbc.so
+ && rm mysql-odbc.rpm mysql-odbc.rpm.tgz \
+ && ODBC_DIR=$(dpkg -L odbc-postgresql | rg '^/usr/lib/.*-linux-gnu/odbc$') \
+ && ln -s /usr/lib64/libmyodbc8a.so "$ODBC_DIR" \
+ && ln -s /usr/lib64/libmyodbc8a.so "$ODBC_DIR"/libmyodbc.so
# Unfortunately this is required for a single test for conversion data from zookeeper to clickhouse-keeper.
# ZooKeeper is not started by default, but consumes some space in containers.
diff --git a/docker/test/integration/helper_container/Dockerfile b/docker/test/integration/helper_container/Dockerfile
index 6a093081bf2..60adaea1796 100644
--- a/docker/test/integration/helper_container/Dockerfile
+++ b/docker/test/integration/helper_container/Dockerfile
@@ -2,4 +2,7 @@
# Helper docker container to run iptables without sudo
FROM alpine
-RUN apk add -U iproute2
+RUN apk add --no-cache -U iproute2 \
+ && for bin in iptables iptables-restore iptables-save; \
+ do ln -sf xtables-nft-multi "/sbin/$bin"; \
+ done
diff --git a/docker/test/integration/mysql_php_client/Dockerfile b/docker/test/integration/mysql_php_client/Dockerfile
index 55db4d15a7f..0e11ae023e6 100644
--- a/docker/test/integration/mysql_php_client/Dockerfile
+++ b/docker/test/integration/mysql_php_client/Dockerfile
@@ -1,7 +1,7 @@
# docker build -t clickhouse/mysql-php-client .
# MySQL PHP client docker container
-FROM php:8.0.18-cli
+FROM php:8-cli-alpine
COPY ./client.crt client.crt
COPY ./client.key client.key
diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile
index 14c97e479f6..8e95d94b6dc 100644
--- a/docker/test/integration/runner/Dockerfile
+++ b/docker/test/integration/runner/Dockerfile
@@ -1,5 +1,5 @@
# docker build -t clickhouse/integration-tests-runner .
-FROM ubuntu:20.04
+FROM ubuntu:22.04
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
@@ -47,26 +47,30 @@ ENV TZ=Etc/UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
ENV DOCKER_CHANNEL stable
+# Unpin the docker version after the release 24.0.3 is released
+# https://github.com/moby/moby/issues/45770#issuecomment-1618255130
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
&& add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" \
&& apt-get update \
&& env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
- docker-ce \
+ docker-ce='5:23.*' \
&& rm -rf \
/var/lib/apt/lists/* \
/var/cache/debconf \
/tmp/* \
- && apt-get clean
+ && apt-get clean \
+ && dockerd --version; docker --version
-RUN dockerd --version; docker --version
RUN python3 -m pip install --no-cache-dir \
PyMySQL \
- aerospike==4.0.0 \
- avro==1.10.2 \
+ aerospike==11.1.0 \
asyncio \
+ avro==1.10.2 \
+ azure-storage-blob \
cassandra-driver \
- confluent-kafka==1.5.0 \
+ confluent-kafka==1.9.2 \
+ delta-spark==2.3.0 \
dict2xml \
dicttoxml \
docker \
@@ -76,47 +80,52 @@ RUN python3 -m pip install --no-cache-dir \
kafka-python \
kazoo \
lz4 \
+ meilisearch==0.18.3 \
minio \
nats-py \
protobuf \
- psycopg2-binary==2.8.6 \
+ psycopg2-binary==2.9.6 \
+ pyhdfs \
pymongo==3.11.0 \
+ pyspark==3.3.2 \
pytest \
pytest-order==1.0.0 \
- pytest-timeout \
pytest-random \
- pytest-xdist \
pytest-repeat \
+ pytest-timeout \
+ pytest-xdist \
pytz \
redis \
- tzlocal==2.1 \
- urllib3 \
requests-kerberos \
- pyspark==3.3.2 \
- delta-spark==2.2.0 \
- pyhdfs \
- azure-storage-blob \
- meilisearch==0.18.3
-
-COPY modprobe.sh /usr/local/bin/modprobe
-COPY dockerd-entrypoint.sh /usr/local/bin/
-COPY compose/ /compose/
-COPY misc/ /misc/
+ tzlocal==2.1 \
+ retry \
+ urllib3
+# Hudi supports only spark 3.3.*, not 3.4
RUN curl -fsSL -O https://dlcdn.apache.org/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz \
&& tar xzvf spark-3.3.2-bin-hadoop3.tgz -C / \
&& rm spark-3.3.2-bin-hadoop3.tgz
# download spark and packages
# if you change packages, don't forget to update them in tests/integration/helpers/cluster.py
-RUN echo ":quit" | /spark-3.3.2-bin-hadoop3/bin/spark-shell --packages "org.apache.hudi:hudi-spark3.3-bundle_2.12:0.13.0,io.delta:delta-core_2.12:2.2.0,org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.1.0" > /dev/null
+RUN packages="org.apache.hudi:hudi-spark3.3-bundle_2.12:0.13.0,\
+io.delta:delta-core_2.12:2.3.0,\
+org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.1.0" \
+ && /spark-3.3.2-bin-hadoop3/bin/spark-shell --packages "$packages" > /dev/null \
+ && find /root/.ivy2/ -name '*.jar' -exec ln -sf {} /spark-3.3.2-bin-hadoop3/jars/ \;
RUN set -x \
&& addgroup --system dockremap \
- && adduser --system dockremap \
+ && adduser --system dockremap \
&& adduser dockremap dockremap \
&& echo 'dockremap:165536:65536' >> /etc/subuid \
- && echo 'dockremap:165536:65536' >> /etc/subgid
+ && echo 'dockremap:165536:65536' >> /etc/subgid
+
+COPY modprobe.sh /usr/local/bin/modprobe
+COPY dockerd-entrypoint.sh /usr/local/bin/
+COPY compose/ /compose/
+COPY misc/ /misc/
+
# Same options as in test/base/Dockerfile
# (in case you need to override them in tests)
@@ -126,4 +135,5 @@ ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'
EXPOSE 2375
ENTRYPOINT ["dockerd-entrypoint.sh"]
-CMD ["sh", "-c", "pytest $PYTEST_OPTS"]
+# To pass additional arguments (i.e. list of tests) use PYTEST_ADDOPTS
+CMD ["sh", "-c", "pytest"]
diff --git a/docker/test/integration/runner/compose/docker_compose_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kafka.yml
index 7e34f4c114d..30d1b0bed3f 100644
--- a/docker/test/integration/runner/compose/docker_compose_kafka.yml
+++ b/docker/test/integration/runner/compose/docker_compose_kafka.yml
@@ -4,6 +4,8 @@ services:
kafka_zookeeper:
image: zookeeper:3.4.9
hostname: kafka_zookeeper
+ ports:
+ - 2181:2181
environment:
ZOO_MY_ID: 1
ZOO_PORT: 2181
@@ -15,15 +17,14 @@ services:
image: confluentinc/cp-kafka:5.2.0
hostname: kafka1
ports:
- - ${KAFKA_EXTERNAL_PORT:-8081}:${KAFKA_EXTERNAL_PORT:-8081}
+ - ${KAFKA_EXTERNAL_PORT}:${KAFKA_EXTERNAL_PORT}
environment:
KAFKA_ADVERTISED_LISTENERS: INSIDE://localhost:${KAFKA_EXTERNAL_PORT},OUTSIDE://kafka1:19092
KAFKA_ADVERTISED_HOST_NAME: kafka1
- KAFKA_LISTENERS: INSIDE://0.0.0.0:${KAFKA_EXTERNAL_PORT},OUTSIDE://0.0.0.0:19092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
KAFKA_BROKER_ID: 1
- KAFKA_ZOOKEEPER_CONNECT: "kafka_zookeeper:2181"
+ KAFKA_ZOOKEEPER_CONNECT: kafka_zookeeper:2181
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
depends_on:
@@ -35,13 +36,38 @@ services:
image: confluentinc/cp-schema-registry:5.2.0
hostname: schema-registry
ports:
- - ${SCHEMA_REGISTRY_EXTERNAL_PORT:-12313}:${SCHEMA_REGISTRY_INTERNAL_PORT:-12313}
+ - ${SCHEMA_REGISTRY_EXTERNAL_PORT}:${SCHEMA_REGISTRY_EXTERNAL_PORT}
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
+ SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:${SCHEMA_REGISTRY_EXTERNAL_PORT}
+ SCHEMA_REGISTRY_SCHEMA_REGISTRY_GROUP_ID: noauth
depends_on:
- kafka_zookeeper
- kafka1
+ restart: always
+ security_opt:
+ - label:disable
+
+ schema-registry-auth:
+ image: confluentinc/cp-schema-registry:5.2.0
+ hostname: schema-registry-auth
+ ports:
+ - ${SCHEMA_REGISTRY_AUTH_EXTERNAL_PORT}:${SCHEMA_REGISTRY_AUTH_EXTERNAL_PORT}
+ environment:
+ SCHEMA_REGISTRY_HOST_NAME: schema-registry-auth
+ SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:${SCHEMA_REGISTRY_AUTH_EXTERNAL_PORT}
+ SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
+ SCHEMA_REGISTRY_AUTHENTICATION_METHOD: BASIC
+ SCHEMA_REGISTRY_AUTHENTICATION_ROLES: user
+ SCHEMA_REGISTRY_AUTHENTICATION_REALM: RealmFooBar
+ SCHEMA_REGISTRY_OPTS: "-Djava.security.auth.login.config=/etc/schema-registry/secrets/schema_registry_jaas.conf"
+ SCHEMA_REGISTRY_SCHEMA_REGISTRY_GROUP_ID: auth
+ volumes:
+ - ${SCHEMA_REGISTRY_DIR:-}/secrets:/etc/schema-registry/secrets
+ depends_on:
+ - kafka_zookeeper
+ - kafka1
+ restart: always
security_opt:
- label:disable
diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh
index fe47fc90951..3c4ff522b36 100755
--- a/docker/test/integration/runner/dockerd-entrypoint.sh
+++ b/docker/test/integration/runner/dockerd-entrypoint.sh
@@ -12,6 +12,17 @@ echo '{
"registry-mirrors" : ["http://dockerhub-proxy.dockerhub-proxy-zone:5000"]
}' | dd of=/etc/docker/daemon.json 2>/dev/null
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+ # move the processes from the root group to the /init group,
+ # otherwise writing subtree_control fails with EBUSY.
+ # An error during moving non-existent process (i.e., "cat") is ignored.
+ mkdir -p /sys/fs/cgroup/init
+ xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+ # enable controllers
+ sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
+ > /sys/fs/cgroup/cgroup.subtree_control
+fi
+
# In case of test hung it is convenient to use pytest --pdb to debug it,
# and on hung you can simply press Ctrl-C and it will spawn a python pdb,
# but on SIGINT dockerd will exit, so ignore it to preserve the daemon.
@@ -52,6 +63,8 @@ export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=/clickhouse-config
export CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH=/clickhouse-odbc-bridge
export CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH=/clickhouse-library-bridge
+export DOCKER_BASE_TAG=${DOCKER_BASE_TAG:=latest}
+export DOCKER_HELPER_TAG=${DOCKER_HELPER_TAG:=latest}
export DOCKER_MYSQL_GOLANG_CLIENT_TAG=${DOCKER_MYSQL_GOLANG_CLIENT_TAG:=latest}
export DOCKER_DOTNET_CLIENT_TAG=${DOCKER_DOTNET_CLIENT_TAG:=latest}
export DOCKER_MYSQL_JAVA_CLIENT_TAG=${DOCKER_MYSQL_JAVA_CLIENT_TAG:=latest}
diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh
index 293ad9ac411..798d2a40b12 100755
--- a/docker/test/performance-comparison/compare.sh
+++ b/docker/test/performance-comparison/compare.sh
@@ -14,6 +14,13 @@ LEFT_SERVER_PORT=9001
# patched version
RIGHT_SERVER_PORT=9002
+# abort_conf -- abort if some options is not recognized
+# abort -- abort if something is not right in the env (i.e. per-cpu arenas does not work)
+# narenas -- set them explicitly to avoid disabling per-cpu arena in env
+# that returns different number of CPUs for some of the following
+# _SC_NPROCESSORS_ONLN/_SC_NPROCESSORS_CONF/sched_getaffinity
+export MALLOC_CONF="abort_conf:true,abort:true,narenas:$(nproc --all)"
+
function wait_for_server # port, pid
{
for _ in {1..60}
@@ -109,10 +116,6 @@ function restart
while pkill -f clickhouse-serv ; do echo . ; sleep 1 ; done
echo all killed
- # Change the jemalloc settings here.
- # https://github.com/jemalloc/jemalloc/wiki/Getting-Started
- export MALLOC_CONF="confirm_conf:true"
-
set -m # Spawn servers in their own process groups
local left_server_opts=(
@@ -147,8 +150,6 @@ function restart
set +m
- unset MALLOC_CONF
-
wait_for_server $LEFT_SERVER_PORT $left_pid
echo left ok
diff --git a/docker/test/sqlancer/process_sqlancer_result.py b/docker/test/sqlancer/process_sqlancer_result.py
index 3bed4578565..06abb52abf8 100755
--- a/docker/test/sqlancer/process_sqlancer_result.py
+++ b/docker/test/sqlancer/process_sqlancer_result.py
@@ -16,7 +16,6 @@ def process_result(result_folder):
"TLPGroupBy",
"TLPHaving",
"TLPWhere",
- "TLPWhereGroupBy",
"NoREC",
]
failed_tests = []
diff --git a/docker/test/sqlancer/run.sh b/docker/test/sqlancer/run.sh
index 4a0f0f6a512..b186bc155a2 100755
--- a/docker/test/sqlancer/run.sh
+++ b/docker/test/sqlancer/run.sh
@@ -33,7 +33,7 @@ cd /workspace
for _ in $(seq 1 60); do if [[ $(wget -q 'localhost:8123' -O-) == 'Ok.' ]]; then break ; else sleep 1; fi ; done
-cd /sqlancer/sqlancer-master
+cd /sqlancer/sqlancer-main
TIMEOUT=300
NUM_QUERIES=1000
diff --git a/docker/test/sqllogic/Dockerfile b/docker/test/sqllogic/Dockerfile
index 83dcf7e1f56..5cf71e4d3f8 100644
--- a/docker/test/sqllogic/Dockerfile
+++ b/docker/test/sqllogic/Dockerfile
@@ -13,6 +13,7 @@ RUN apt-get update --yes \
sqlite3 \
unixodbc \
unixodbc-dev \
+ odbcinst \
sudo \
&& apt-get clean
diff --git a/docker/test/sqllogic/run.sh b/docker/test/sqllogic/run.sh
index 8d0252e3c98..444252837a3 100755
--- a/docker/test/sqllogic/run.sh
+++ b/docker/test/sqllogic/run.sh
@@ -92,8 +92,8 @@ sudo clickhouse stop ||:
for _ in $(seq 1 60); do if [[ $(wget --timeout=1 -q 'localhost:8123' -O-) == 'Ok.' ]]; then sleep 1 ; else break; fi ; done
-grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
-pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz &
+rg -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
+zstd < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst &
# Compressed (FIXME: remove once only github actions will be left)
rm /var/log/clickhouse-server/clickhouse-server.log
diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile
index 71a2e92e3a8..f513735a2d0 100644
--- a/docker/test/stateful/Dockerfile
+++ b/docker/test/stateful/Dockerfile
@@ -16,8 +16,9 @@ COPY s3downloader /s3downloader
ENV S3_URL="https://clickhouse-datasets.s3.amazonaws.com"
ENV DATASETS="hits visits"
-RUN npm install -g azurite
-RUN npm install tslib
+# The following is already done in clickhouse/stateless-test
+# RUN npm install -g azurite
+# RUN npm install tslib
COPY run.sh /
CMD ["/bin/bash", "/run.sh"]
diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile
index 40109255a7e..e1e84c427ba 100644
--- a/docker/test/stateless/Dockerfile
+++ b/docker/test/stateless/Dockerfile
@@ -20,6 +20,7 @@ RUN apt-get update -y \
netcat-openbsd \
nodejs \
npm \
+ odbcinst \
openjdk-11-jre-headless \
openssl \
postgresql-client \
@@ -32,7 +33,6 @@ RUN apt-get update -y \
qemu-user-static \
sqlite3 \
sudo \
- telnet \
tree \
unixodbc \
wget \
@@ -71,7 +71,7 @@ RUN arch=${TARGETARCH:-amd64} \
&& chmod +x ./mc ./minio
-RUN wget 'https://dlcdn.apache.org/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
+RUN wget --no-verbose 'https://dlcdn.apache.org/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
&& tar -xvf hadoop-3.3.1.tar.gz \
&& rm -rf hadoop-3.3.1.tar.gz
@@ -79,8 +79,8 @@ ENV MINIO_ROOT_USER="clickhouse"
ENV MINIO_ROOT_PASSWORD="clickhouse"
ENV EXPORT_S3_STORAGE_POLICIES=1
-RUN npm install -g azurite
-RUN npm install tslib
+RUN npm install -g azurite \
+ && npm install -g tslib
COPY run.sh /
COPY setup_minio.sh /
diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 21cb3168083..3694fb7c2f6 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -4,6 +4,9 @@
set -e -x -a
# Choose random timezone for this test run.
+#
+# NOTE: that clickhouse-test will randomize session_timezone by itself as well
+# (it will choose between default server timezone and something specific).
TZ="$(rg -v '#' /usr/share/zoneinfo/zone.tab | awk '{print $3}' | shuf | head -n1)"
echo "Choosen random timezone $TZ"
ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" > /etc/timezone
@@ -18,6 +21,9 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
# shellcheck disable=SC1091
source /usr/share/clickhouse-test/ci/attach_gdb.lib || true # FIXME: to not break old builds, clean on 2023-09-01
+# shellcheck disable=SC1091
+source /usr/share/clickhouse-test/ci/utils.lib || true # FIXME: to not break old builds, clean on 2023-09-01
+
# install test configs
/usr/share/clickhouse-test/config/install.sh
@@ -90,6 +96,22 @@ sleep 5
attach_gdb_to_clickhouse || true # FIXME: to not break old builds, clean on 2023-09-01
+function fn_exists() {
+ declare -F "$1" > /dev/null;
+}
+
+# FIXME: to not break old builds, clean on 2023-09-01
+function try_run_with_retry() {
+ local total_retries="$1"
+ shift
+
+ if fn_exists run_with_retry; then
+ run_with_retry "$total_retries" "$@"
+ else
+ "$@"
+ fi
+}
+
function run_tests()
{
set -x
@@ -137,8 +159,7 @@ function run_tests()
ADDITIONAL_OPTIONS+=('--report-logs-stats')
- clickhouse-test "00001_select_1" > /dev/null ||:
- clickhouse-client -q "insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')" ||:
+ try_run_with_retry 10 clickhouse-client -q "insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')"
set +e
clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile
index e9712f430fd..eddeb04758b 100644
--- a/docker/test/stress/Dockerfile
+++ b/docker/test/stress/Dockerfile
@@ -8,8 +8,6 @@ RUN apt-get update -y \
apt-get install --yes --no-install-recommends \
bash \
tzdata \
- fakeroot \
- debhelper \
parallel \
expect \
python3 \
@@ -20,7 +18,6 @@ RUN apt-get update -y \
sudo \
openssl \
netcat-openbsd \
- telnet \
brotli \
&& apt-get clean
diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh
index 4926967d2d2..9217fcfddd9 100644
--- a/docker/test/stress/run.sh
+++ b/docker/test/stress/run.sh
@@ -14,6 +14,7 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
# Stress tests and upgrade check uses similar code that was placed
# in a separate bash library. See tests/ci/stress_tests.lib
+source /usr/share/clickhouse-test/ci/attach_gdb.lib
source /usr/share/clickhouse-test/ci/stress_tests.lib
install_packages package_folder
@@ -52,7 +53,7 @@ azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
start
-shellcheck disable=SC2086 # No quotes because I want to split it into words.
+# shellcheck disable=SC2086 # No quotes because I want to split it into words.
/s3downloader --url-prefix "$S3_URL" --dataset-names $DATASETS
chmod 777 -R /var/lib/clickhouse
clickhouse-client --query "ATTACH DATABASE IF NOT EXISTS datasets ENGINE = Ordinary"
diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile
index 746cc7bb2d5..2aa0b1a62d6 100644
--- a/docker/test/style/Dockerfile
+++ b/docker/test/style/Dockerfile
@@ -18,7 +18,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
python3-pip \
shellcheck \
yamllint \
- && pip3 install black==23.1.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \
+ && pip3 install black==23.1.0 boto3 codespell==2.2.1 mypy==1.3.0 PyGithub unidiff pylint==2.6.2 \
&& apt-get clean \
&& rm -rf /root/.cache/pip
diff --git a/docker/test/upgrade/Dockerfile b/docker/test/upgrade/Dockerfile
index 8e5890b81a0..9152230af1c 100644
--- a/docker/test/upgrade/Dockerfile
+++ b/docker/test/upgrade/Dockerfile
@@ -8,8 +8,6 @@ RUN apt-get update -y \
apt-get install --yes --no-install-recommends \
bash \
tzdata \
- fakeroot \
- debhelper \
parallel \
expect \
python3 \
@@ -20,7 +18,6 @@ RUN apt-get update -y \
sudo \
openssl \
netcat-openbsd \
- telnet \
brotli \
&& apt-get clean
diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh
index 951c443c30d..13c352d5d41 100644
--- a/docker/test/upgrade/run.sh
+++ b/docker/test/upgrade/run.sh
@@ -16,6 +16,7 @@ ln -s /usr/share/clickhouse-test/ci/get_previous_release_tag.py /usr/bin/get_pre
# Stress tests and upgrade check uses similar code that was placed
# in a separate bash library. See tests/ci/stress_tests.lib
+source /usr/share/clickhouse-test/ci/attach_gdb.lib
source /usr/share/clickhouse-test/ci/stress_tests.lib
azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
@@ -59,12 +60,22 @@ install_packages previous_release_package_folder
# available for dump via clickhouse-local
configure
+# it contains some new settings, but we can safely remove it
+rm /etc/clickhouse-server/config.d/merge_tree.xml
+rm /etc/clickhouse-server/config.d/enable_wait_for_shutdown_replicated_tables.xml
rm /etc/clickhouse-server/users.d/nonconst_timezone.xml
start
stop
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.initial.log
+# Start server from previous release
+# Let's enable S3 storage by default
+export USE_S3_STORAGE_FOR_MERGE_TREE=1
+# Previous version may not be ready for fault injections
+export ZOOKEEPER_FAULT_INJECTION=0
+configure
+
# force_sync=false doesn't work correctly on some older versions
sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
| sed "s|false|true|" \
@@ -74,17 +85,14 @@ sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-serv
# But we still need default disk because some tables loaded only into it
sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
| sed "s|s3|s3default|" \
- > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
+ > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp
+mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
-# Start server from previous release
-# Let's enable S3 storage by default
-export USE_S3_STORAGE_FOR_MERGE_TREE=1
-# Previous version may not be ready for fault injections
-export ZOOKEEPER_FAULT_INJECTION=0
-configure
-
+# it contains some new settings, but we can safely remove it
+rm /etc/clickhouse-server/config.d/merge_tree.xml
+rm /etc/clickhouse-server/config.d/enable_wait_for_shutdown_replicated_tables.xml
rm /etc/clickhouse-server/users.d/nonconst_timezone.xml
start
@@ -115,6 +123,13 @@ mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/c
install_packages package_folder
export ZOOKEEPER_FAULT_INJECTION=1
configure
+
+# Just in case previous version left some garbage in zk
+sudo cat /etc/clickhouse-server/config.d/lost_forever_check.xml \
+ | sed "s|>1<|>0<|g" \
+ > /etc/clickhouse-server/config.d/lost_forever_check.xml.tmp
+sudo mv /etc/clickhouse-server/config.d/lost_forever_check.xml.tmp /etc/clickhouse-server/config.d/lost_forever_check.xml
+
start 500
clickhouse-client --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \
|| (rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt \
@@ -178,6 +193,7 @@ rg -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
-e "Authentication failed" \
-e "Cannot flush" \
-e "Container already exists" \
+ -e "doesn't have metadata version on disk" \
clickhouse-server.upgrade.log \
| grep -av -e "_repl_01111_.*Mapping for table with UUID" \
| zgrep -Fa "" > /test_output/upgrade_error_messages.txt \
diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile
index a49278e960b..359041eed03 100644
--- a/docker/test/util/Dockerfile
+++ b/docker/test/util/Dockerfile
@@ -1,5 +1,5 @@
# docker build -t clickhouse/test-util .
-FROM ubuntu:20.04
+FROM ubuntu:22.04
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
@@ -44,7 +44,6 @@ RUN apt-get update \
clang-${LLVM_VERSION} \
clang-tidy-${LLVM_VERSION} \
cmake \
- fakeroot \
gdb \
git \
gperf \
@@ -94,7 +93,10 @@ RUN mkdir /tmp/ccache \
&& rm -rf /tmp/ccache
ARG TARGETARCH
-ARG SCCACHE_VERSION=v0.4.1
+ARG SCCACHE_VERSION=v0.5.4
+ENV SCCACHE_IGNORE_SERVER_IO_ERROR=1
+# sccache requires a value for the region. So by default we use The Default Region
+ENV SCCACHE_REGION=us-east-1
RUN arch=${TARGETARCH:-amd64} \
&& case $arch in \
amd64) rarch=x86_64 ;; \
diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py
index c75a3500831..fd4cc9f4bf7 100755
--- a/docker/test/util/process_functional_tests_result.py
+++ b/docker/test/util/process_functional_tests_result.py
@@ -86,7 +86,7 @@ def process_test_log(log_path, broken_tests):
test_name,
"NOT_FAILED",
test_time,
- ["This test passed. Update broken_tests.txt.\n"],
+ ["This test passed. Update analyzer_tech_debt.txt.\n"],
)
)
else:
@@ -205,7 +205,7 @@ if __name__ == "__main__":
parser.add_argument("--in-results-dir", default="/test_output/")
parser.add_argument("--out-results-file", default="/test_output/test_results.tsv")
parser.add_argument("--out-status-file", default="/test_output/check_status.tsv")
- parser.add_argument("--broken-tests", default="/broken_tests.txt")
+ parser.add_argument("--broken-tests", default="/analyzer_tech_debt.txt")
args = parser.parse_args()
broken_tests = list()
diff --git a/docs/_description_templates/template-data-type.md b/docs/_description_templates/template-data-type.md
new file mode 100644
index 00000000000..239edb2808b
--- /dev/null
+++ b/docs/_description_templates/template-data-type.md
@@ -0,0 +1,29 @@
+---
+toc_priority:
+toc_title:
+---
+
+# data_type_name {#data_type-name}
+
+Description.
+
+**Parameters** (Optional)
+
+- `x` — Description. [Type name](relative/path/to/type/dscr.md#type).
+- `y` — Description. [Type name](relative/path/to/type/dscr.md#type).
+
+**Examples**
+
+```sql
+
+```
+
+## Additional Info {#additional-info} (Optional)
+
+The name of an additional section can be any, for example, **Usage**.
+
+**See Also** (Optional)
+
+- [link](#)
+
+[Original article](https://clickhouse.com/docs/en/data-types//)
diff --git a/docs/_description_templates/template-engine.md b/docs/_description_templates/template-engine.md
new file mode 100644
index 00000000000..392bc59ed33
--- /dev/null
+++ b/docs/_description_templates/template-engine.md
@@ -0,0 +1,63 @@
+# EngineName {#enginename}
+
+- What the Database/Table engine does.
+- Relations with other engines if they exist.
+
+## Creating a Database {#creating-a-database}
+``` sql
+ CREATE DATABASE ...
+```
+or
+
+## Creating a Table {#creating-a-table}
+``` sql
+ CREATE TABLE ...
+```
+
+**Engine Parameters**
+
+**Query Clauses** (for Table engines only)
+
+## Virtual columns {#virtual-columns} (for Table engines only)
+
+List and virtual columns with description, if they exist.
+
+## Data Types Support {#data_types-support} (for Database engines only)
+
+| EngineName | ClickHouse |
+|-----------------------|------------------------------------|
+| NativeDataTypeName | [ClickHouseDataTypeName](link#) |
+
+
+## Specifics and recommendations {#specifics-and-recommendations}
+
+Algorithms
+Specifics of read and write processes
+Examples of tasks
+Recommendations for usage
+Specifics of data storage
+
+## Usage Example {#usage-example}
+
+The example must show usage and use cases. The following text contains the recommended parts of this section.
+
+Input table:
+
+``` text
+```
+
+Query:
+
+``` sql
+```
+
+Result:
+
+``` text
+```
+
+Follow up with any text to clarify the example.
+
+**See Also**
+
+- [link](#)
diff --git a/docs/_description_templates/template-function.md b/docs/_description_templates/template-function.md
new file mode 100644
index 00000000000..6bdc764c449
--- /dev/null
+++ b/docs/_description_templates/template-function.md
@@ -0,0 +1,51 @@
+## functionName {#functionname-in-lower-case}
+
+Short description.
+
+**Syntax** (without SELECT)
+
+``` sql
+
+```
+
+Alias: ``. (Optional)
+
+More text (Optional).
+
+**Arguments** (Optional)
+
+- `x` — Description. Optional (only for optional arguments). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type).
+- `y` — Description. Optional (only for optional arguments). Possible values: .Default value: . [Type name](relative/path/to/type/dscr.md#type).
+
+**Parameters** (Optional, only for parametric aggregate functions)
+
+- `z` — Description. Optional (only for optional parameters). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type).
+
+**Returned value(s)**
+
+- Returned values list.
+
+Type: [Type name](relative/path/to/type/dscr.md#type).
+
+**Example**
+
+The example must show usage and/or a use cases. The following text contains recommended parts of an example.
+
+Input table (Optional):
+
+``` text
+```
+
+Query:
+
+``` sql
+```
+
+Result:
+
+``` text
+```
+
+**See Also** (Optional)
+
+- [link](#)
diff --git a/docs/_description_templates/template-server-setting.md b/docs/_description_templates/template-server-setting.md
new file mode 100644
index 00000000000..0b37d46cf41
--- /dev/null
+++ b/docs/_description_templates/template-server-setting.md
@@ -0,0 +1,33 @@
+## server_setting_name {#server_setting_name}
+
+Description.
+
+Describe what is configured in this section of settings.
+
+Possible value: ...
+
+Default value: ...
+
+**Settings** (Optional)
+
+If the section contains several settings, list them here. Specify possible values and default values:
+
+- setting_1 — Description.
+- setting_2 — Description.
+
+**Example**
+
+```xml
+
+ ...
+ ...
+
+```
+
+**Additional Info** (Optional)
+
+The name of an additional section can be any, for example, **Usage**.
+
+**See Also** (Optional)
+
+- [link](#)
diff --git a/docs/_description_templates/template-setting.md b/docs/_description_templates/template-setting.md
new file mode 100644
index 00000000000..fc912aba3e1
--- /dev/null
+++ b/docs/_description_templates/template-setting.md
@@ -0,0 +1,27 @@
+## setting_name {#setting_name}
+
+Description.
+
+For the switch setting, use the typical phrase: “Enables or disables something …”.
+
+Possible values:
+
+*For switcher setting:*
+
+- 0 — Disabled.
+- 1 — Enabled.
+
+*For another setting (typical phrases):*
+
+- Positive integer.
+- 0 — Disabled or unlimited or something else.
+
+Default value: `value`.
+
+**Additional Info** (Optional)
+
+The name of an additional section can be any, for example, **Usage**.
+
+**See Also** (Optional)
+
+- [link](#)
diff --git a/docs/_description_templates/template-statement.md b/docs/_description_templates/template-statement.md
new file mode 100644
index 00000000000..238570c2217
--- /dev/null
+++ b/docs/_description_templates/template-statement.md
@@ -0,0 +1,24 @@
+# Statement name (for example, SHOW USER) {#statement-name-in-lower-case}
+
+Brief description of what the statement does.
+
+**Syntax**
+
+```sql
+Syntax of the statement.
+```
+
+## Other necessary sections of the description (Optional) {#anchor}
+
+Examples of descriptions with a complicated structure:
+
+- https://clickhouse.com/docs/en/sql-reference/statements/grant/
+- https://clickhouse.com/docs/en/sql-reference/statements/revoke/
+- https://clickhouse.com/docs/en/sql-reference/statements/select/join/
+
+
+**See Also** (Optional)
+
+Links to related topics as a list.
+
+- [link](#)
diff --git a/docs/_description_templates/template-system-table.md b/docs/_description_templates/template-system-table.md
new file mode 100644
index 00000000000..f2decc4bb6d
--- /dev/null
+++ b/docs/_description_templates/template-system-table.md
@@ -0,0 +1,25 @@
+# system.table_name {#system-tables_table-name}
+
+Description.
+
+Columns:
+
+- `column_name` ([data_type_name](path/to/data_type.md)) — Description.
+
+**Example**
+
+Query:
+
+``` sql
+SELECT * FROM system.table_name
+```
+
+Result:
+
+``` text
+Some output. It shouldn't be too long.
+```
+
+**See Also**
+
+- [Article name](path/to/article_name.md) — Some words about referenced information.
diff --git a/docs/_includes/install/universal.sh b/docs/_includes/install/universal.sh
index 1699be138c8..5d4571aed9e 100755
--- a/docs/_includes/install/universal.sh
+++ b/docs/_includes/install/universal.sh
@@ -33,6 +33,9 @@ then
elif [ "${ARCH}" = "powerpc64le" -o "${ARCH}" = "ppc64le" ]
then
DIR="powerpc64le"
+ elif [ "${ARCH}" = "riscv64" ]
+ then
+ DIR="riscv64"
fi
elif [ "${OS}" = "FreeBSD" ]
then
diff --git a/docs/changelogs/v22.8.19.10-lts.md b/docs/changelogs/v22.8.19.10-lts.md
new file mode 100644
index 00000000000..b3990c74e46
--- /dev/null
+++ b/docs/changelogs/v22.8.19.10-lts.md
@@ -0,0 +1,19 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v22.8.19.10-lts (989bc2fe8b0) FIXME as compared to v22.8.18.31-lts (4de7a95a544)
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix subtly broken copy-on-write of ColumnLowCardinality dictionary [#51064](https://github.com/ClickHouse/ClickHouse/pull/51064) ([Michael Kolupaev](https://github.com/al13n321)).
+* Generate safe IVs [#51086](https://github.com/ClickHouse/ClickHouse/pull/51086) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Fix a versions' tweak for tagged commits, improve version_helper [#51035](https://github.com/ClickHouse/ClickHouse/pull/51035) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Sqlancer has changed master to main [#51060](https://github.com/ClickHouse/ClickHouse/pull/51060) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
diff --git a/docs/changelogs/v22.8.20.11-lts.md b/docs/changelogs/v22.8.20.11-lts.md
new file mode 100644
index 00000000000..bd45ce9319a
--- /dev/null
+++ b/docs/changelogs/v22.8.20.11-lts.md
@@ -0,0 +1,20 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v22.8.20.11-lts (c9ca79e24e8) FIXME as compared to v22.8.19.10-lts (989bc2fe8b0)
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix broken index analysis when binary operator contains a null constant argument [#50177](https://github.com/ClickHouse/ClickHouse/pull/50177) ([Amos Bird](https://github.com/amosbird)).
+* Fix incorrect constant folding [#50536](https://github.com/ClickHouse/ClickHouse/pull/50536) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix fuzzer failure in ActionsDAG [#51301](https://github.com/ClickHouse/ClickHouse/pull/51301) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix segfault in MathUnary [#51499](https://github.com/ClickHouse/ClickHouse/pull/51499) ([Ilya Yatsishin](https://github.com/qoega)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Decoupled commits from [#51180](https://github.com/ClickHouse/ClickHouse/issues/51180) for backports [#51561](https://github.com/ClickHouse/ClickHouse/pull/51561) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
diff --git a/docs/changelogs/v23.3.4.17-lts.md b/docs/changelogs/v23.3.4.17-lts.md
new file mode 100644
index 00000000000..41d9d7dd06c
--- /dev/null
+++ b/docs/changelogs/v23.3.4.17-lts.md
@@ -0,0 +1,22 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.3.4.17-lts (2c99b73ff40) FIXME as compared to v23.3.3.52-lts (cb963c474db)
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix crash when Pool::Entry::disconnect() is called [#50334](https://github.com/ClickHouse/ClickHouse/pull/50334) ([Val Doroshchuk](https://github.com/valbok)).
+* Avoid storing logs in Keeper containing unknown operation [#50751](https://github.com/ClickHouse/ClickHouse/pull/50751) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix subtly broken copy-on-write of ColumnLowCardinality dictionary [#51064](https://github.com/ClickHouse/ClickHouse/pull/51064) ([Michael Kolupaev](https://github.com/al13n321)).
+* Generate safe IVs [#51086](https://github.com/ClickHouse/ClickHouse/pull/51086) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Don't mark a part as broken on `Poco::TimeoutException` [#50811](https://github.com/ClickHouse/ClickHouse/pull/50811) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix a versions' tweak for tagged commits, improve version_helper [#51035](https://github.com/ClickHouse/ClickHouse/pull/51035) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Sqlancer has changed master to main [#51060](https://github.com/ClickHouse/ClickHouse/pull/51060) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
diff --git a/docs/changelogs/v23.3.5.9-lts.md b/docs/changelogs/v23.3.5.9-lts.md
new file mode 100644
index 00000000000..df1aab541ec
--- /dev/null
+++ b/docs/changelogs/v23.3.5.9-lts.md
@@ -0,0 +1,19 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.3.5.9-lts (f5fbc2fd2b3) FIXME as compared to v23.3.4.17-lts (2c99b73ff40)
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix broken index analysis when binary operator contains a null constant argument [#50177](https://github.com/ClickHouse/ClickHouse/pull/50177) ([Amos Bird](https://github.com/amosbird)).
+* Cleanup moving parts [#50489](https://github.com/ClickHouse/ClickHouse/pull/50489) ([vdimir](https://github.com/vdimir)).
+* Do not apply projection if read-in-order was enabled. [#50923](https://github.com/ClickHouse/ClickHouse/pull/50923) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Increase max array size in group bitmap [#50620](https://github.com/ClickHouse/ClickHouse/pull/50620) ([Kruglov Pavel](https://github.com/Avogar)).
+
diff --git a/docs/changelogs/v23.3.6.7-lts.md b/docs/changelogs/v23.3.6.7-lts.md
new file mode 100644
index 00000000000..387cc126aba
--- /dev/null
+++ b/docs/changelogs/v23.3.6.7-lts.md
@@ -0,0 +1,19 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.3.6.7-lts (7e3f0a271b7) FIXME as compared to v23.3.5.9-lts (f5fbc2fd2b3)
+
+#### Improvement
+* Backported in [#51240](https://github.com/ClickHouse/ClickHouse/issues/51240): Improve the progress bar for file/s3/hdfs/url table functions by using chunk size from source data and using incremental total size counting in each thread. Fix the progress bar for *Cluster functions. This closes [#47250](https://github.com/ClickHouse/ClickHouse/issues/47250). [#51088](https://github.com/ClickHouse/ClickHouse/pull/51088) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#51529](https://github.com/ClickHouse/ClickHouse/issues/51529): Split huge `RUN` in Dockerfile into smaller conditional. Install the necessary tools on demand in the same `RUN` layer, and remove them after that. Upgrade the OS only once at the beginning. Use a modern way to check the signed repository. Downgrade the base repo to ubuntu:20.04 to address the issues on older docker versions. Upgrade golang version to address golang vulnerabilities. [#51504](https://github.com/ClickHouse/ClickHouse/pull/51504) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix type of LDAP server params hash in cache entry [#50865](https://github.com/ClickHouse/ClickHouse/pull/50865) ([Julian Maicher](https://github.com/jmaicher)).
+
diff --git a/docs/changelogs/v23.3.7.5-lts.md b/docs/changelogs/v23.3.7.5-lts.md
new file mode 100644
index 00000000000..7a5fd5a19b6
--- /dev/null
+++ b/docs/changelogs/v23.3.7.5-lts.md
@@ -0,0 +1,16 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.3.7.5-lts (bc683c11c92) FIXME as compared to v23.3.6.7-lts (7e3f0a271b7)
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#51568](https://github.com/ClickHouse/ClickHouse/issues/51568): This a follow-up for [#51504](https://github.com/ClickHouse/ClickHouse/issues/51504), the cleanup was lost during refactoring. [#51564](https://github.com/ClickHouse/ClickHouse/pull/51564) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix fuzzer failure in ActionsDAG [#51301](https://github.com/ClickHouse/ClickHouse/pull/51301) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
diff --git a/docs/changelogs/v23.3.8.21-lts.md b/docs/changelogs/v23.3.8.21-lts.md
new file mode 100644
index 00000000000..83b5070ef52
--- /dev/null
+++ b/docs/changelogs/v23.3.8.21-lts.md
@@ -0,0 +1,23 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.3.8.21-lts (1675f2264f3) FIXME as compared to v23.3.7.5-lts (bc683c11c92)
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix backward compatibility for IP types hashing in aggregate functions [#50551](https://github.com/ClickHouse/ClickHouse/pull/50551) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix segfault in MathUnary [#51499](https://github.com/ClickHouse/ClickHouse/pull/51499) ([Ilya Yatsishin](https://github.com/qoega)).
+* Fix for moving 'IN' conditions to PREWHERE [#51610](https://github.com/ClickHouse/ClickHouse/pull/51610) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix reading from empty column in `parseSipHashKey` [#51804](https://github.com/ClickHouse/ClickHouse/pull/51804) ([Nikita Taranov](https://github.com/nickitat)).
+* Check refcount in `RemoveManyObjectStorageOperation::finalize` instead of `execute` [#51954](https://github.com/ClickHouse/ClickHouse/pull/51954) ([vdimir](https://github.com/vdimir)).
+* Allow parametric UDFs [#51964](https://github.com/ClickHouse/ClickHouse/pull/51964) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Decoupled commits from [#51180](https://github.com/ClickHouse/ClickHouse/issues/51180) for backports [#51561](https://github.com/ClickHouse/ClickHouse/pull/51561) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix MergeTreeMarksLoader segfaulting if marks file is longer than expected [#51636](https://github.com/ClickHouse/ClickHouse/pull/51636) ([Michael Kolupaev](https://github.com/al13n321)).
+
diff --git a/docs/changelogs/v23.4.4.16-stable.md b/docs/changelogs/v23.4.4.16-stable.md
new file mode 100644
index 00000000000..72056f72091
--- /dev/null
+++ b/docs/changelogs/v23.4.4.16-stable.md
@@ -0,0 +1,22 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.4.4.16-stable (747ba4fc6a0) FIXME as compared to v23.4.3.48-stable (d9199f8d3cc)
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix crash when Pool::Entry::disconnect() is called [#50334](https://github.com/ClickHouse/ClickHouse/pull/50334) ([Val Doroshchuk](https://github.com/valbok)).
+* Fix iceberg V2 optional metadata parsing [#50974](https://github.com/ClickHouse/ClickHouse/pull/50974) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix subtly broken copy-on-write of ColumnLowCardinality dictionary [#51064](https://github.com/ClickHouse/ClickHouse/pull/51064) ([Michael Kolupaev](https://github.com/al13n321)).
+* Generate safe IVs [#51086](https://github.com/ClickHouse/ClickHouse/pull/51086) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Don't mark a part as broken on `Poco::TimeoutException` [#50811](https://github.com/ClickHouse/ClickHouse/pull/50811) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix a versions' tweak for tagged commits, improve version_helper [#51035](https://github.com/ClickHouse/ClickHouse/pull/51035) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Sqlancer has changed master to main [#51060](https://github.com/ClickHouse/ClickHouse/pull/51060) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
diff --git a/docs/changelogs/v23.4.5.22-stable.md b/docs/changelogs/v23.4.5.22-stable.md
new file mode 100644
index 00000000000..2d61f5b11cf
--- /dev/null
+++ b/docs/changelogs/v23.4.5.22-stable.md
@@ -0,0 +1,27 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.4.5.22-stable (0ced5d6a8da) FIXME as compared to v23.4.4.16-stable (747ba4fc6a0)
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#51530](https://github.com/ClickHouse/ClickHouse/issues/51530): Split huge `RUN` in Dockerfile into smaller conditional. Install the necessary tools on demand in the same `RUN` layer, and remove them after that. Upgrade the OS only once at the beginning. Use a modern way to check the signed repository. Downgrade the base repo to ubuntu:20.04 to address the issues on older docker versions. Upgrade golang version to address golang vulnerabilities. [#51504](https://github.com/ClickHouse/ClickHouse/pull/51504) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#51570](https://github.com/ClickHouse/ClickHouse/issues/51570): This a follow-up for [#51504](https://github.com/ClickHouse/ClickHouse/issues/51504), the cleanup was lost during refactoring. [#51564](https://github.com/ClickHouse/ClickHouse/pull/51564) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix broken index analysis when binary operator contains a null constant argument [#50177](https://github.com/ClickHouse/ClickHouse/pull/50177) ([Amos Bird](https://github.com/amosbird)).
+* Fix reconnecting of HTTPS session when target host IP was changed [#50240](https://github.com/ClickHouse/ClickHouse/pull/50240) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Fix incorrect constant folding [#50536](https://github.com/ClickHouse/ClickHouse/pull/50536) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix type of LDAP server params hash in cache entry [#50865](https://github.com/ClickHouse/ClickHouse/pull/50865) ([Julian Maicher](https://github.com/jmaicher)).
+* Fallback to parsing big integer from String instead of exception in Parquet format [#50873](https://github.com/ClickHouse/ClickHouse/pull/50873) ([Kruglov Pavel](https://github.com/Avogar)).
+* Do not apply projection if read-in-order was enabled. [#50923](https://github.com/ClickHouse/ClickHouse/pull/50923) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix fuzzer failure in ActionsDAG [#51301](https://github.com/ClickHouse/ClickHouse/pull/51301) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Increase max array size in group bitmap [#50620](https://github.com/ClickHouse/ClickHouse/pull/50620) ([Kruglov Pavel](https://github.com/Avogar)).
+
diff --git a/docs/changelogs/v23.4.6.25-stable.md b/docs/changelogs/v23.4.6.25-stable.md
new file mode 100644
index 00000000000..01a9c06f3e9
--- /dev/null
+++ b/docs/changelogs/v23.4.6.25-stable.md
@@ -0,0 +1,26 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.4.6.25-stable (a06848b1770) FIXME as compared to v23.4.5.22-stable (0ced5d6a8da)
+
+#### Improvement
+* Backported in [#51234](https://github.com/ClickHouse/ClickHouse/issues/51234): Improve the progress bar for file/s3/hdfs/url table functions by using chunk size from source data and using incremental total size counting in each thread. Fix the progress bar for *Cluster functions. This closes [#47250](https://github.com/ClickHouse/ClickHouse/issues/47250). [#51088](https://github.com/ClickHouse/ClickHouse/pull/51088) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix backward compatibility for IP types hashing in aggregate functions [#50551](https://github.com/ClickHouse/ClickHouse/pull/50551) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix segfault in MathUnary [#51499](https://github.com/ClickHouse/ClickHouse/pull/51499) ([Ilya Yatsishin](https://github.com/qoega)).
+* Fix for moving 'IN' conditions to PREWHERE [#51610](https://github.com/ClickHouse/ClickHouse/pull/51610) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix reading from empty column in `parseSipHashKey` [#51804](https://github.com/ClickHouse/ClickHouse/pull/51804) ([Nikita Taranov](https://github.com/nickitat)).
+* Allow parametric UDFs [#51964](https://github.com/ClickHouse/ClickHouse/pull/51964) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Decoupled commits from [#51180](https://github.com/ClickHouse/ClickHouse/issues/51180) for backports [#51561](https://github.com/ClickHouse/ClickHouse/pull/51561) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix MergeTreeMarksLoader segfaulting if marks file is longer than expected [#51636](https://github.com/ClickHouse/ClickHouse/pull/51636) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix source image for sqllogic [#51728](https://github.com/ClickHouse/ClickHouse/pull/51728) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
diff --git a/docs/changelogs/v23.5.3.24-stable.md b/docs/changelogs/v23.5.3.24-stable.md
new file mode 100644
index 00000000000..967a50b3b0e
--- /dev/null
+++ b/docs/changelogs/v23.5.3.24-stable.md
@@ -0,0 +1,26 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.5.3.24-stable (76f54616d3b) FIXME as compared to v23.5.2.7-stable (5751aa1ab9f)
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix Log family table return wrong rows count after truncate [#50585](https://github.com/ClickHouse/ClickHouse/pull/50585) ([flynn](https://github.com/ucasfl)).
+* Fix bug in `uniqExact` parallel merging [#50590](https://github.com/ClickHouse/ClickHouse/pull/50590) ([Nikita Taranov](https://github.com/nickitat)).
+* Revert recent grace hash join changes [#50699](https://github.com/ClickHouse/ClickHouse/pull/50699) ([vdimir](https://github.com/vdimir)).
+* Avoid storing logs in Keeper containing unknown operation [#50751](https://github.com/ClickHouse/ClickHouse/pull/50751) ([Antonio Andelic](https://github.com/antonio2368)).
+* Add compat setting for non-const timezones [#50834](https://github.com/ClickHouse/ClickHouse/pull/50834) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix iceberg V2 optional metadata parsing [#50974](https://github.com/ClickHouse/ClickHouse/pull/50974) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix subtly broken copy-on-write of ColumnLowCardinality dictionary [#51064](https://github.com/ClickHouse/ClickHouse/pull/51064) ([Michael Kolupaev](https://github.com/al13n321)).
+* Generate safe IVs [#51086](https://github.com/ClickHouse/ClickHouse/pull/51086) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Don't mark a part as broken on `Poco::TimeoutException` [#50811](https://github.com/ClickHouse/ClickHouse/pull/50811) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix a versions' tweak for tagged commits, improve version_helper [#51035](https://github.com/ClickHouse/ClickHouse/pull/51035) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Sqlancer has changed master to main [#51060](https://github.com/ClickHouse/ClickHouse/pull/51060) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
diff --git a/docs/changelogs/v23.5.4.25-stable.md b/docs/changelogs/v23.5.4.25-stable.md
new file mode 100644
index 00000000000..53d3a7c9c0a
--- /dev/null
+++ b/docs/changelogs/v23.5.4.25-stable.md
@@ -0,0 +1,31 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.5.4.25-stable (190f962abcf) FIXME as compared to v23.5.3.24-stable (76f54616d3b)
+
+#### Improvement
+* Backported in [#51235](https://github.com/ClickHouse/ClickHouse/issues/51235): Improve the progress bar for file/s3/hdfs/url table functions by using chunk size from source data and using incremental total size counting in each thread. Fix the progress bar for *Cluster functions. This closes [#47250](https://github.com/ClickHouse/ClickHouse/issues/47250). [#51088](https://github.com/ClickHouse/ClickHouse/pull/51088) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#51255](https://github.com/ClickHouse/ClickHouse/issues/51255): Disable cache setting `do_not_evict_index_and_mark_files` (Was enabled in `23.5`). [#51222](https://github.com/ClickHouse/ClickHouse/pull/51222) ([Kseniia Sumarokova](https://github.com/kssenii)).
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#51531](https://github.com/ClickHouse/ClickHouse/issues/51531): Split huge `RUN` in Dockerfile into smaller conditional. Install the necessary tools on demand in the same `RUN` layer, and remove them after that. Upgrade the OS only once at the beginning. Use a modern way to check the signed repository. Downgrade the base repo to ubuntu:20.04 to address the issues on older docker versions. Upgrade golang version to address golang vulnerabilities. [#51504](https://github.com/ClickHouse/ClickHouse/pull/51504) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Backported in [#51572](https://github.com/ClickHouse/ClickHouse/issues/51572): This a follow-up for [#51504](https://github.com/ClickHouse/ClickHouse/issues/51504), the cleanup was lost during refactoring. [#51564](https://github.com/ClickHouse/ClickHouse/pull/51564) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Query Cache: Try to fix bad cast from ColumnConst to ColumnVector [#50704](https://github.com/ClickHouse/ClickHouse/pull/50704) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix type of LDAP server params hash in cache entry [#50865](https://github.com/ClickHouse/ClickHouse/pull/50865) ([Julian Maicher](https://github.com/jmaicher)).
+* Fallback to parsing big integer from String instead of exception in Parquet format [#50873](https://github.com/ClickHouse/ClickHouse/pull/50873) ([Kruglov Pavel](https://github.com/Avogar)).
+* Do not apply projection if read-in-order was enabled. [#50923](https://github.com/ClickHouse/ClickHouse/pull/50923) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix race azure blob storage iterator [#50936](https://github.com/ClickHouse/ClickHouse/pull/50936) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix ineffective query cache for SELECTs with subqueries [#51132](https://github.com/ClickHouse/ClickHouse/pull/51132) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix fuzzer failure in ActionsDAG [#51301](https://github.com/ClickHouse/ClickHouse/pull/51301) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Fix ParallelReadBuffer seek [#50820](https://github.com/ClickHouse/ClickHouse/pull/50820) ([Michael Kolupaev](https://github.com/al13n321)).
+
diff --git a/docs/changelogs/v23.6.1.1524-stable.md b/docs/changelogs/v23.6.1.1524-stable.md
new file mode 100644
index 00000000000..6d295d61ef4
--- /dev/null
+++ b/docs/changelogs/v23.6.1.1524-stable.md
@@ -0,0 +1,301 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.6.1.1524-stable (d1c7e13d088) FIXME as compared to v23.5.1.3174-stable (2fec796e73e)
+
+#### Backward Incompatible Change
+* Delete feature `do_not_evict_index_and_mark_files` in the fs cache. This feature was only making things worse. [#51253](https://github.com/ClickHouse/ClickHouse/pull/51253) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Remove ALTER support for experimental LIVE VIEW. [#51287](https://github.com/ClickHouse/ClickHouse/pull/51287) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### New Feature
+* Add setting `session_timezone`, it is used as default timezone for session when not explicitly specified. [#44149](https://github.com/ClickHouse/ClickHouse/pull/44149) ([Andrey Zvonov](https://github.com/zvonand)).
+* Added overlay database engine and representation of a directory as a database This commit adds 4 databases: 1. DatabaseOverlay: Implements the IDatabase interface. Allow to combine multiple databases, such as FileSystem and Memory. Internally, it stores a vector with other database pointers and proxies requests to them in turn until it is executed successfully. 2. DatabaseFilesystem: allows to read-only interact with files stored on the file system. Internally, it uses TableFunctionFile to implicitly load file when a user requests the table. Result of TableFunctionFile call cached inside to provide quick access. 3. DatabaseS3: allows to read-only interact with s3 storage. It uses TableFunctionS3 to implicitly load table from s3 4. DatabaseHDFS: allows to interact with hdfs storage. It uses TableFunctionHDFS to implicitly load table from hdfs. [#48821](https://github.com/ClickHouse/ClickHouse/pull/48821) ([alekseygolub](https://github.com/alekseygolub)).
+* Add a new setting named `use_mysql_types_in_show_columns` to alter the `SHOW COLUMNS` SQL statement to display MySQL equivalent types when a client is connected via the MySQL compatibility port. [#49577](https://github.com/ClickHouse/ClickHouse/pull/49577) ([Thomas Panetti](https://github.com/tpanetti)).
+* Added option `--rename_files_after_processing `. This closes [#34207](https://github.com/ClickHouse/ClickHouse/issues/34207). [#49626](https://github.com/ClickHouse/ClickHouse/pull/49626) ([alekseygolub](https://github.com/alekseygolub)).
+* 1. Add `TableFunctionRedis` 3. Add table engine Redis 4. Add `RedisCommon` which contains Redis related tools and types 5. Support `equals` and `in` filter push down into Redis. [#50150](https://github.com/ClickHouse/ClickHouse/pull/50150) ([JackyWoo](https://github.com/JackyWoo)).
+* Allow to skip empty files in file/s3/url/hdfs table functions using settings `s3_skip_empty_files`, `hdfs_skip_empty_files`, `engine_file_skip_empty_files`, `engine_url_skip_empty_files`. [#50364](https://github.com/ClickHouse/ClickHouse/pull/50364) ([Kruglov Pavel](https://github.com/Avogar)).
+* Clickhouse-client can now be called with a connection instead of "--host", "--port", "--user" etc. [#50689](https://github.com/ClickHouse/ClickHouse/pull/50689) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
+* Codec DEFLATE_QPL is now controlled via server setting "enable_deflate_qpl_codec" (default: false) instead of setting "allow_experimental_codecs". This marks QPL_DEFLATE non-experimental. [#50775](https://github.com/ClickHouse/ClickHouse/pull/50775) ([Robert Schulze](https://github.com/rschu1ze)).
+
+#### Performance Improvement
+* Improve performance with enabled QueryProfiler using thread-local timer_id instead of global object. [#48778](https://github.com/ClickHouse/ClickHouse/pull/48778) ([Jiebin Sun](https://github.com/jiebinn)).
+* Rewrite CapnProto input/output format to improve its performance. Map column names and CapnProto fields case insensitive, fix reading/writing of nested structure fields. [#49752](https://github.com/ClickHouse/ClickHouse/pull/49752) ([Kruglov Pavel](https://github.com/Avogar)).
+* Optimize parquet write performance for parallel threads. [#50102](https://github.com/ClickHouse/ClickHouse/pull/50102) ([Hongbin Ma](https://github.com/binmahone)).
+* ### Documentation entry for user-facing changes Disable `parallelize_output_from_storages` for processing MATERIALIZED VIEWs and storages with one block only. [#50214](https://github.com/ClickHouse/ClickHouse/pull/50214) ([Azat Khuzhin](https://github.com/azat)).
+* Merge PR https://github.com/ClickHouse/ClickHouse/pull/46558 (Avoid processing already sorted data). Avoid block permutation during sort if the block is already sorted. [#50697](https://github.com/ClickHouse/ClickHouse/pull/50697) ([Maksim Kita](https://github.com/kitaisreal)).
+* In the earlier PRs ([#50062](https://github.com/ClickHouse/ClickHouse/issues/50062), [#50307](https://github.com/ClickHouse/ClickHouse/issues/50307)), we used to propose an optimization pattern which transforms the predicates with toYear/toYYYYMM into its equivalent but converter-free form. This transformation could bring significant performance impact to some workloads, such as SSB. However, as issue [#50628](https://github.com/ClickHouse/ClickHouse/issues/50628) indicated, these two PRs would introduce some issues which may results in incomplete query results, and as a result, they were reverted by [#50629](https://github.com/ClickHouse/ClickHouse/issues/50629). [#50951](https://github.com/ClickHouse/ClickHouse/pull/50951) ([Zhiguo Zhou](https://github.com/ZhiguoZh)).
+* Make multiple list requests to ZooKeeper in parallel to speed up reading from system.zookeeper table. [#51042](https://github.com/ClickHouse/ClickHouse/pull/51042) ([Alexander Gololobov](https://github.com/davenger)).
+* Speedup initialization of DateTime lookup tables for time zones. This should reduce startup/connect time of clickhouse client especially in debug build as it is rather heavy. [#51347](https://github.com/ClickHouse/ClickHouse/pull/51347) ([Alexander Gololobov](https://github.com/davenger)).
+
+#### Improvement
+* Allow to cast IPv6 to IPv4 address for CIDR ::ffff:0:0/96 (IPv4-mapped addresses). [#49759](https://github.com/ClickHouse/ClickHouse/pull/49759) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Update MongoDB protocol to support MongoDB 5.1 version and newer. Support for the versions with the old protocol (<3.6) is preserved. Closes [#45621](https://github.com/ClickHouse/ClickHouse/issues/45621), [#49879](https://github.com/ClickHouse/ClickHouse/issues/49879). [#50061](https://github.com/ClickHouse/ClickHouse/pull/50061) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Improved scheduling of merge selecting and cleanup tasks in `ReplicatedMergeTree`. The tasks will not be executed too frequently when there's nothing to merge or cleanup. Added settings `max_merge_selecting_sleep_ms`, `merge_selecting_sleep_slowdown_factor`, `max_cleanup_delay_period` and `cleanup_thread_preferred_points_per_iteration`. It should close [#31919](https://github.com/ClickHouse/ClickHouse/issues/31919). [#50107](https://github.com/ClickHouse/ClickHouse/pull/50107) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Support parallel replicas with the analyzer. [#50441](https://github.com/ClickHouse/ClickHouse/pull/50441) ([Raúl Marín](https://github.com/Algunenano)).
+* Add setting `input_format_max_bytes_to_read_for_schema_inference` to limit the number of bytes to read in schema inference. Closes [#50577](https://github.com/ClickHouse/ClickHouse/issues/50577). [#50592](https://github.com/ClickHouse/ClickHouse/pull/50592) ([Kruglov Pavel](https://github.com/Avogar)).
+* Respect setting input_format_as_default in schema inference. [#50602](https://github.com/ClickHouse/ClickHouse/pull/50602) ([Kruglov Pavel](https://github.com/Avogar)).
+* Make filter push down through cross join. [#50605](https://github.com/ClickHouse/ClickHouse/pull/50605) ([Han Fei](https://github.com/hanfei1991)).
+* Actual lz4 version is used now. [#50621](https://github.com/ClickHouse/ClickHouse/pull/50621) ([Nikita Taranov](https://github.com/nickitat)).
+* Allow to skip trailing empty lines in CSV/TSV/CustomSeparated formats via settings `input_format_csv_skip_trailing_empty_lines`, `input_format_tsv_skip_trailing_empty_lines` and `input_format_custom_skip_trailing_empty_lines` (disabled by default). Closes [#49315](https://github.com/ClickHouse/ClickHouse/issues/49315). [#50635](https://github.com/ClickHouse/ClickHouse/pull/50635) ([Kruglov Pavel](https://github.com/Avogar)).
+* Functions "toDateOrDefault|OrNull()" and "accuateCast[OrDefault|OrNull]()" now correctly parse numeric arguments. [#50709](https://github.com/ClickHouse/ClickHouse/pull/50709) ([Dmitry Kardymon](https://github.com/kardymonds)).
+* Currently, the csv input format can not parse the csv file with whitespace or \t field delimiter, and these delimiters is supported in spark. [#50712](https://github.com/ClickHouse/ClickHouse/pull/50712) ([KevinyhZou](https://github.com/KevinyhZou)).
+* Settings `number_of_mutations_to_delay` and `number_of_mutations_to_throw` are enabled by default now with values 500 and 1000 respectively. [#50726](https://github.com/ClickHouse/ClickHouse/pull/50726) ([Anton Popov](https://github.com/CurtizJ)).
+* Keeper improvement: add feature flags for Keeper API. Each feature flag can be disabled or enabled by defining it under `keeper_server.feature_flags` config. E.g. to enable `CheckNotExists` request, `keeper_server.feature_flags.check_not_exists` should be set to `1` on Keeper. [#50796](https://github.com/ClickHouse/ClickHouse/pull/50796) ([Antonio Andelic](https://github.com/antonio2368)).
+* The dashboard correctly shows missing values. This closes [#50831](https://github.com/ClickHouse/ClickHouse/issues/50831). [#50832](https://github.com/ClickHouse/ClickHouse/pull/50832) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* CGroups metrics related to CPU are replaced with one metric, `CGroupMaxCPU` for better usability. The `Normalized` CPU usage metrics will be normalized to CGroups limits instead of the total number of CPUs when they are set. This closes [#50836](https://github.com/ClickHouse/ClickHouse/issues/50836). [#50835](https://github.com/ClickHouse/ClickHouse/pull/50835) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Relax the thresholds for "too many parts" to be more modern. Return the backpressure during long-running insert queries. [#50856](https://github.com/ClickHouse/ClickHouse/pull/50856) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Added the possibility to use date and time arguments in syslog timestamp format in functions parseDateTimeBestEffort*() and parseDateTime64BestEffort*(). [#50925](https://github.com/ClickHouse/ClickHouse/pull/50925) ([Victor Krasnov](https://github.com/sirvickr)).
+* Suggest using `APPEND` or `TRUNCATE` for `INTO OUTFILE` when file exists. [#50950](https://github.com/ClickHouse/ClickHouse/pull/50950) ([alekar](https://github.com/alekar)).
+* Add embedded keeper-client to standalone keeper binary. [#50964](https://github.com/ClickHouse/ClickHouse/pull/50964) ([pufit](https://github.com/pufit)).
+* Command line parameter "--password" in clickhouse-client can now be specified only once. [#50966](https://github.com/ClickHouse/ClickHouse/pull/50966) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
+* Fix data lakes slowness because of synchronous head requests. (Related to Iceberg/Deltalake/Hudi being slow with a lot of files). [#50976](https://github.com/ClickHouse/ClickHouse/pull/50976) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Use `hash_of_all_files` from `system.parts` to check identity of parts during on-cluster backups. [#50997](https://github.com/ClickHouse/ClickHouse/pull/50997) ([Vitaly Baranov](https://github.com/vitlibar)).
+* The system table zookeeper_connection connected_time identifies the time when the connection is established (standard format), and session_uptime_elapsed_seconds is added, which labels the duration of the established connection session (in seconds). [#51026](https://github.com/ClickHouse/ClickHouse/pull/51026) ([郭小龙](https://github.com/guoxiaolongzte)).
+* Show halves of checksums in `system.parts`, `system.projection_parts` and in error messages in the correct order. [#51040](https://github.com/ClickHouse/ClickHouse/pull/51040) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Do not replicate `ALTER PARTITION` queries and mutations through `Replicated` database if it has only one shard and the underlying table is `ReplicatedMergeTree`. [#51049](https://github.com/ClickHouse/ClickHouse/pull/51049) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Improve the progress bar for file/s3/hdfs/url table functions by using chunk size from source data and using incremental total size counting in each thread. Fix the progress bar for *Cluster functions. This closes [#47250](https://github.com/ClickHouse/ClickHouse/issues/47250). [#51088](https://github.com/ClickHouse/ClickHouse/pull/51088) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add total_bytes_to_read to Progress packet in TCP protocol for better Progress bar. [#51158](https://github.com/ClickHouse/ClickHouse/pull/51158) ([Kruglov Pavel](https://github.com/Avogar)).
+* Better checking of data parts on disks with filesystem cache. [#51164](https://github.com/ClickHouse/ClickHouse/pull/51164) ([Anton Popov](https://github.com/CurtizJ)).
+* Disable cache setting `do_not_evict_index_and_mark_files` (Was enabled in `23.5`). [#51222](https://github.com/ClickHouse/ClickHouse/pull/51222) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix sometimes not correct current_elements_num in fs cache. [#51242](https://github.com/ClickHouse/ClickHouse/pull/51242) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add random sleep before merges/mutations execution to split load more evenly between replicas in case of zero-copy replication. [#51282](https://github.com/ClickHouse/ClickHouse/pull/51282) ([alesapin](https://github.com/alesapin)).
+* The function `transform` as well as `CASE` with value matching started to support all data types. This closes [#29730](https://github.com/ClickHouse/ClickHouse/issues/29730). This closes [#32387](https://github.com/ClickHouse/ClickHouse/issues/32387). This closes [#50827](https://github.com/ClickHouse/ClickHouse/issues/50827). This closes [#31336](https://github.com/ClickHouse/ClickHouse/issues/31336). This closes [#40493](https://github.com/ClickHouse/ClickHouse/issues/40493). [#51351](https://github.com/ClickHouse/ClickHouse/pull/51351) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* We have found a bug in LLVM that makes the usage of `compile_expressions` setting unsafe. It is disabled by default. [#51368](https://github.com/ClickHouse/ClickHouse/pull/51368) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Issue [#50220](https://github.com/ClickHouse/ClickHouse/issues/50220) reports a core in `grace_hash` join. We finally reproduce the exception on local, and found that the issue is related to the failure of creating temporary file. Somehow this is triggered in https://github.com/ClickHouse/ClickHouse/pull/49816 https://github.com/ClickHouse/ClickHouse/pull/49483. [#51382](https://github.com/ClickHouse/ClickHouse/pull/51382) ([lgbo](https://github.com/lgbo-ustc)).
+
+#### Build/Testing/Packaging Improvement
+* Update contrib/re2 to 2023-06-02. [#50949](https://github.com/ClickHouse/ClickHouse/pull/50949) ([Yuriy Chernyshov](https://github.com/georgthegreat)).
+* ClickHouse server will print the list of changed settings on fatal errors. This closes [#51137](https://github.com/ClickHouse/ClickHouse/issues/51137). [#51138](https://github.com/ClickHouse/ClickHouse/pull/51138) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* In https://github.com/ClickHouse/ClickHouse/pull/51143 the fasstests failed, but the status wasn't created because of the chown `file not found`. This addresses it. Decrease the default values for `http-max-field-value-size` and `http_max_field_name_size` to 128K. [#51163](https://github.com/ClickHouse/ClickHouse/pull/51163) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Update Ubuntu version in docker containers. [#51180](https://github.com/ClickHouse/ClickHouse/pull/51180) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Allow building ClickHouse with clang-17. [#51300](https://github.com/ClickHouse/ClickHouse/pull/51300) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* [SQLancer](https://github.com/sqlancer/sqlancer) check is considered stable as bugs that were triggered by it are fixed. Now failures of SQLancer check will be reported as failed check status. [#51340](https://github.com/ClickHouse/ClickHouse/pull/51340) ([Ilya Yatsishin](https://github.com/qoega)).
+* Making our CI even better. [#51494](https://github.com/ClickHouse/ClickHouse/pull/51494) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Split huge `RUN` in Dockerfile into smaller conditional. Install the necessary tools on demand in the same `RUN` layer, and remove them after that. Upgrade the OS only once at the beginning. Use a modern way to check the signed repository. Downgrade the base repo to ubuntu:20.04 to address the issues on older docker versions. Upgrade golang version to address golang vulnerabilities. [#51504](https://github.com/ClickHouse/ClickHouse/pull/51504) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* This a follow-up for [#51504](https://github.com/ClickHouse/ClickHouse/issues/51504), the cleanup was lost during refactoring. [#51564](https://github.com/ClickHouse/ClickHouse/pull/51564) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Report loading status for executable dictionaries correctly [#48775](https://github.com/ClickHouse/ClickHouse/pull/48775) ([Anton Kozlov](https://github.com/tonickkozlov)).
+* Proper mutation of skip indices and projections [#50104](https://github.com/ClickHouse/ClickHouse/pull/50104) ([Amos Bird](https://github.com/amosbird)).
+* Cleanup moving parts [#50489](https://github.com/ClickHouse/ClickHouse/pull/50489) ([vdimir](https://github.com/vdimir)).
+* Fix backward compatibility for IP types hashing in aggregate functions [#50551](https://github.com/ClickHouse/ClickHouse/pull/50551) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix Log family table return wrong rows count after truncate [#50585](https://github.com/ClickHouse/ClickHouse/pull/50585) ([flynn](https://github.com/ucasfl)).
+* Fix bug in `uniqExact` parallel merging [#50590](https://github.com/ClickHouse/ClickHouse/pull/50590) ([Nikita Taranov](https://github.com/nickitat)).
+* Revert recent grace hash join changes [#50699](https://github.com/ClickHouse/ClickHouse/pull/50699) ([vdimir](https://github.com/vdimir)).
+* Query Cache: Try to fix bad cast from ColumnConst to ColumnVector [#50704](https://github.com/ClickHouse/ClickHouse/pull/50704) ([Robert Schulze](https://github.com/rschu1ze)).
+* Do not read all the columns from right GLOBAL JOIN table. [#50721](https://github.com/ClickHouse/ClickHouse/pull/50721) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Avoid storing logs in Keeper containing unknown operation [#50751](https://github.com/ClickHouse/ClickHouse/pull/50751) ([Antonio Andelic](https://github.com/antonio2368)).
+* SummingMergeTree support for DateTime64 [#50797](https://github.com/ClickHouse/ClickHouse/pull/50797) ([Jordi Villar](https://github.com/jrdi)).
+* Add compat setting for non-const timezones [#50834](https://github.com/ClickHouse/ClickHouse/pull/50834) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix type of LDAP server params hash in cache entry [#50865](https://github.com/ClickHouse/ClickHouse/pull/50865) ([Julian Maicher](https://github.com/jmaicher)).
+* Fallback to parsing big integer from String instead of exception in Parquet format [#50873](https://github.com/ClickHouse/ClickHouse/pull/50873) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix checking the lock file too often while writing a backup [#50889](https://github.com/ClickHouse/ClickHouse/pull/50889) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Do not apply projection if read-in-order was enabled. [#50923](https://github.com/ClickHouse/ClickHouse/pull/50923) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix race azure blob storage iterator [#50936](https://github.com/ClickHouse/ClickHouse/pull/50936) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix erroneous `sort_description` propagation in `CreatingSets` [#50955](https://github.com/ClickHouse/ClickHouse/pull/50955) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix iceberg V2 optional metadata parsing [#50974](https://github.com/ClickHouse/ClickHouse/pull/50974) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* MaterializedMySQL: Keep parentheses for empty table overrides [#50977](https://github.com/ClickHouse/ClickHouse/pull/50977) ([Val Doroshchuk](https://github.com/valbok)).
+* Fix crash in BackupCoordinationStageSync::setError() [#51012](https://github.com/ClickHouse/ClickHouse/pull/51012) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix subtly broken copy-on-write of ColumnLowCardinality dictionary [#51064](https://github.com/ClickHouse/ClickHouse/pull/51064) ([Michael Kolupaev](https://github.com/al13n321)).
+* Generate safe IVs [#51086](https://github.com/ClickHouse/ClickHouse/pull/51086) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Fix ineffective query cache for SELECTs with subqueries [#51132](https://github.com/ClickHouse/ClickHouse/pull/51132) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix Set index with constant nullable comparison. [#51205](https://github.com/ClickHouse/ClickHouse/pull/51205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix a crash in s3 and s3Cluster functions [#51209](https://github.com/ClickHouse/ClickHouse/pull/51209) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix core dump when compile expression [#51231](https://github.com/ClickHouse/ClickHouse/pull/51231) ([LiuNeng](https://github.com/liuneng1994)).
+* Fix use-after-free in StorageURL when switching URLs [#51260](https://github.com/ClickHouse/ClickHouse/pull/51260) ([Michael Kolupaev](https://github.com/al13n321)).
+* Updated check for parameterized view [#51272](https://github.com/ClickHouse/ClickHouse/pull/51272) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix multiple writing of same file to backup [#51299](https://github.com/ClickHouse/ClickHouse/pull/51299) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix fuzzer failure in ActionsDAG [#51301](https://github.com/ClickHouse/ClickHouse/pull/51301) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Remove garbage from function `transform` [#51350](https://github.com/ClickHouse/ClickHouse/pull/51350) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix MSan report in lowerUTF8/upperUTF8 [#51371](https://github.com/ClickHouse/ClickHouse/pull/51371) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* fs cache: fix a bit incorrect use_count after [#44985](https://github.com/ClickHouse/ClickHouse/issues/44985) [#51406](https://github.com/ClickHouse/ClickHouse/pull/51406) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix segfault in MathUnary [#51499](https://github.com/ClickHouse/ClickHouse/pull/51499) ([Ilya Yatsishin](https://github.com/qoega)).
+* Fix logical assert in `tupleElement()` with default values [#51534](https://github.com/ClickHouse/ClickHouse/pull/51534) ([Robert Schulze](https://github.com/rschu1ze)).
+* fs cache: remove file from opened file cache immediately when evicting file [#51596](https://github.com/ClickHouse/ClickHouse/pull/51596) ([Kseniia Sumarokova](https://github.com/kssenii)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Deprecate delete-on-destroy.txt [#49181](https://github.com/ClickHouse/ClickHouse/pull/49181) ([Alexander Gololobov](https://github.com/davenger)).
+* Attempt to increase the general runners' survival rate [#49283](https://github.com/ClickHouse/ClickHouse/pull/49283) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Refactor subqueries for IN [#49570](https://github.com/ClickHouse/ClickHouse/pull/49570) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Test plan optimization analyzer [#50095](https://github.com/ClickHouse/ClickHouse/pull/50095) ([Igor Nikonov](https://github.com/devcrafter)).
+* Implement endianness-independent serialization for quantileTiming [#50324](https://github.com/ClickHouse/ClickHouse/pull/50324) ([ltrk2](https://github.com/ltrk2)).
+* require `finalize()` call before d-tor for all writes buffers [#50395](https://github.com/ClickHouse/ClickHouse/pull/50395) ([Sema Checherinda](https://github.com/CheSema)).
+* Implement big-endian support for the deterministic reservoir sampler [#50405](https://github.com/ClickHouse/ClickHouse/pull/50405) ([ltrk2](https://github.com/ltrk2)).
+* Fix compilation error on big-endian platforms [#50406](https://github.com/ClickHouse/ClickHouse/pull/50406) ([ltrk2](https://github.com/ltrk2)).
+* Attach gdb in stateless tests [#50487](https://github.com/ClickHouse/ClickHouse/pull/50487) ([Kruglov Pavel](https://github.com/Avogar)).
+* JIT infrastructure refactoring [#50531](https://github.com/ClickHouse/ClickHouse/pull/50531) ([Maksim Kita](https://github.com/kitaisreal)).
+* Analyzer: Do not apply Query Tree optimizations on shards [#50584](https://github.com/ClickHouse/ClickHouse/pull/50584) ([Dmitry Novik](https://github.com/novikd)).
+* Increase max array size in group bitmap [#50620](https://github.com/ClickHouse/ClickHouse/pull/50620) ([Kruglov Pavel](https://github.com/Avogar)).
+* Misc Annoy index improvements [#50661](https://github.com/ClickHouse/ClickHouse/pull/50661) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix reading negative decimals in avro format [#50668](https://github.com/ClickHouse/ClickHouse/pull/50668) ([Kruglov Pavel](https://github.com/Avogar)).
+* Unify priorities for connection pools [#50675](https://github.com/ClickHouse/ClickHouse/pull/50675) ([Sergei Trifonov](https://github.com/serxa)).
+* Prostpone check of outdated parts [#50676](https://github.com/ClickHouse/ClickHouse/pull/50676) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Unify priorities: `IExecutableTask`s [#50677](https://github.com/ClickHouse/ClickHouse/pull/50677) ([Sergei Trifonov](https://github.com/serxa)).
+* Disable grace_hash join in stress tests [#50693](https://github.com/ClickHouse/ClickHouse/pull/50693) ([vdimir](https://github.com/vdimir)).
+* ReverseTransform small improvement [#50698](https://github.com/ClickHouse/ClickHouse/pull/50698) ([Maksim Kita](https://github.com/kitaisreal)).
+* Support OPTIMIZE for temporary tables [#50710](https://github.com/ClickHouse/ClickHouse/pull/50710) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Refactor reading from object storages [#50711](https://github.com/ClickHouse/ClickHouse/pull/50711) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix data race in log message of cached buffer [#50723](https://github.com/ClickHouse/ClickHouse/pull/50723) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add new keywords into projections documentation [#50743](https://github.com/ClickHouse/ClickHouse/pull/50743) ([YalalovSM](https://github.com/YalalovSM)).
+* Fix build for aarch64 (temporary disable azure) [#50770](https://github.com/ClickHouse/ClickHouse/pull/50770) ([alesapin](https://github.com/alesapin)).
+* Update version after release [#50772](https://github.com/ClickHouse/ClickHouse/pull/50772) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Update version_date.tsv and changelogs after v23.5.1.3174-stable [#50774](https://github.com/ClickHouse/ClickHouse/pull/50774) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update CHANGELOG.md [#50788](https://github.com/ClickHouse/ClickHouse/pull/50788) ([Ilya Yatsishin](https://github.com/qoega)).
+* Update version_date.tsv and changelogs after v23.2.7.32-stable [#50809](https://github.com/ClickHouse/ClickHouse/pull/50809) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Desctructing --> Destructing [#50810](https://github.com/ClickHouse/ClickHouse/pull/50810) ([Robert Schulze](https://github.com/rschu1ze)).
+* Don't mark a part as broken on `Poco::TimeoutException` [#50811](https://github.com/ClickHouse/ClickHouse/pull/50811) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Rename azure_blob_storage to azureBlobStorage [#50812](https://github.com/ClickHouse/ClickHouse/pull/50812) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix ParallelReadBuffer seek [#50820](https://github.com/ClickHouse/ClickHouse/pull/50820) ([Michael Kolupaev](https://github.com/al13n321)).
+* [RFC] Print git hash when crashing [#50823](https://github.com/ClickHouse/ClickHouse/pull/50823) ([Michael Kolupaev](https://github.com/al13n321)).
+* Add tests for function "transform" [#50833](https://github.com/ClickHouse/ClickHouse/pull/50833) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Update version_date.tsv and changelogs after v23.5.2.7-stable [#50844](https://github.com/ClickHouse/ClickHouse/pull/50844) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Updated changelog with azureBlobStorage table function & engine entry [#50850](https://github.com/ClickHouse/ClickHouse/pull/50850) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Update easy_tasks_sorted_ru.md [#50853](https://github.com/ClickHouse/ClickHouse/pull/50853) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Document x86 / ARM prerequisites for Docker image [#50867](https://github.com/ClickHouse/ClickHouse/pull/50867) ([Robert Schulze](https://github.com/rschu1ze)).
+* MaterializedMySQL: Add test_named_collections [#50874](https://github.com/ClickHouse/ClickHouse/pull/50874) ([Val Doroshchuk](https://github.com/valbok)).
+* Update version_date.tsv and changelogs after v22.8.18.31-lts [#50881](https://github.com/ClickHouse/ClickHouse/pull/50881) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelogs after v23.3.3.52-lts [#50882](https://github.com/ClickHouse/ClickHouse/pull/50882) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelogs after v23.4.3.48-stable [#50883](https://github.com/ClickHouse/ClickHouse/pull/50883) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* MaterializedMySQL: Add additional test case to insert_with_modify_binlog_checksum [#50884](https://github.com/ClickHouse/ClickHouse/pull/50884) ([Val Doroshchuk](https://github.com/valbok)).
+* Update broken tests list [#50886](https://github.com/ClickHouse/ClickHouse/pull/50886) ([Dmitry Novik](https://github.com/novikd)).
+* Fix LOGICAL_ERROR in snowflakeToDateTime*() [#50893](https://github.com/ClickHouse/ClickHouse/pull/50893) ([Robert Schulze](https://github.com/rschu1ze)).
+* Tests with parallel replicas are no more "always green" [#50896](https://github.com/ClickHouse/ClickHouse/pull/50896) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Slightly more information in error message about cached disk [#50897](https://github.com/ClickHouse/ClickHouse/pull/50897) ([Michael Kolupaev](https://github.com/al13n321)).
+* do not call finalize after exception [#50907](https://github.com/ClickHouse/ClickHouse/pull/50907) ([Sema Checherinda](https://github.com/CheSema)).
+* Update Annoy docs [#50912](https://github.com/ClickHouse/ClickHouse/pull/50912) ([Robert Schulze](https://github.com/rschu1ze)).
+* A bit safer UserDefinedSQLFunctionVisitor [#50913](https://github.com/ClickHouse/ClickHouse/pull/50913) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Update contribe/orc in .gitmodules [#50920](https://github.com/ClickHouse/ClickHouse/pull/50920) ([San](https://github.com/santrancisco)).
+* MaterializedMySQL: Add missing DROP DATABASE for tests [#50924](https://github.com/ClickHouse/ClickHouse/pull/50924) ([Val Doroshchuk](https://github.com/valbok)).
+* Fix 'Illegal column timezone' in stress tests [#50929](https://github.com/ClickHouse/ClickHouse/pull/50929) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix tests sanity checks and avoid dropping system.query_log table [#50934](https://github.com/ClickHouse/ClickHouse/pull/50934) ([Azat Khuzhin](https://github.com/azat)).
+* Fix tests for throttling by allowing more margin of error for trottling event [#50935](https://github.com/ClickHouse/ClickHouse/pull/50935) ([Azat Khuzhin](https://github.com/azat)).
+* 01746_convert_type_with_default: Temporarily disable flaky test [#50937](https://github.com/ClickHouse/ClickHouse/pull/50937) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix the statless tests image for old commits [#50947](https://github.com/ClickHouse/ClickHouse/pull/50947) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix logic in `AsynchronousBoundedReadBuffer::seek` [#50952](https://github.com/ClickHouse/ClickHouse/pull/50952) ([Nikita Taranov](https://github.com/nickitat)).
+* Uncomment flaky test (01746_convert_type_with_default) [#50954](https://github.com/ClickHouse/ClickHouse/pull/50954) ([Dmitry Kardymon](https://github.com/kardymonds)).
+* Fix keeper-client help message [#50965](https://github.com/ClickHouse/ClickHouse/pull/50965) ([pufit](https://github.com/pufit)).
+* fix build issue on clang 15 [#50967](https://github.com/ClickHouse/ClickHouse/pull/50967) ([Chang chen](https://github.com/baibaichen)).
+* Docs: Fix embedded video link [#50972](https://github.com/ClickHouse/ClickHouse/pull/50972) ([Robert Schulze](https://github.com/rschu1ze)).
+* Change submodule capnproto to it's fork in ClickHouse [#50987](https://github.com/ClickHouse/ClickHouse/pull/50987) ([Kruglov Pavel](https://github.com/Avogar)).
+* Attempt to make 01281_group_by_limit_memory_tracking not flaky [#50995](https://github.com/ClickHouse/ClickHouse/pull/50995) ([Dmitry Novik](https://github.com/novikd)).
+* Fix flaky 02561_null_as_default_more_formats [#51001](https://github.com/ClickHouse/ClickHouse/pull/51001) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix flaky test_seekable_formats [#51002](https://github.com/ClickHouse/ClickHouse/pull/51002) ([Kruglov Pavel](https://github.com/Avogar)).
+* Follow-up to [#50448](https://github.com/ClickHouse/ClickHouse/issues/50448) [#51006](https://github.com/ClickHouse/ClickHouse/pull/51006) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix a versions' tweak for tagged commits, improve version_helper [#51035](https://github.com/ClickHouse/ClickHouse/pull/51035) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Sqlancer has changed master to main [#51060](https://github.com/ClickHouse/ClickHouse/pull/51060) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Do not spam sqlancer build log [#51061](https://github.com/ClickHouse/ClickHouse/pull/51061) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Refactor IColumn::forEachSubcolumn to make it slightly harder to implement incorrectly [#51072](https://github.com/ClickHouse/ClickHouse/pull/51072) ([Michael Kolupaev](https://github.com/al13n321)).
+* MaterializedMySQL: Rename materialize_with_ddl.py -> materialized_with_ddl [#51074](https://github.com/ClickHouse/ClickHouse/pull/51074) ([Val Doroshchuk](https://github.com/valbok)).
+* Improve woboq browser report [#51077](https://github.com/ClickHouse/ClickHouse/pull/51077) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix for part_names_mutex used after destruction [#51099](https://github.com/ClickHouse/ClickHouse/pull/51099) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix ColumnConst::forEachSubcolumn missing from previous PR [#51102](https://github.com/ClickHouse/ClickHouse/pull/51102) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix the test 02783_parsedatetimebesteffort_syslog flakiness [#51112](https://github.com/ClickHouse/ClickHouse/pull/51112) ([Victor Krasnov](https://github.com/sirvickr)).
+* Compatibility with clang-17 [#51114](https://github.com/ClickHouse/ClickHouse/pull/51114) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Make more parallel get requests to ZooKeeper in system.zookeeper [#51118](https://github.com/ClickHouse/ClickHouse/pull/51118) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix 02703_max_local_write_bandwidth flakiness [#51120](https://github.com/ClickHouse/ClickHouse/pull/51120) ([Azat Khuzhin](https://github.com/azat)).
+* Update version_date.tsv and changelogs after v23.5.3.24-stable [#51121](https://github.com/ClickHouse/ClickHouse/pull/51121) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelogs after v23.4.4.16-stable [#51122](https://github.com/ClickHouse/ClickHouse/pull/51122) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelogs after v23.3.4.17-lts [#51123](https://github.com/ClickHouse/ClickHouse/pull/51123) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelogs after v22.8.19.10-lts [#51124](https://github.com/ClickHouse/ClickHouse/pull/51124) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix typo [#51126](https://github.com/ClickHouse/ClickHouse/pull/51126) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Slightly better diagnostics [#51127](https://github.com/ClickHouse/ClickHouse/pull/51127) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Small fix in `MergeTreePrefetchedReadPool` [#51131](https://github.com/ClickHouse/ClickHouse/pull/51131) ([Nikita Taranov](https://github.com/nickitat)).
+* Don't report table function accesses to system.errors [#51147](https://github.com/ClickHouse/ClickHouse/pull/51147) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix SQLancer branch name [#51148](https://github.com/ClickHouse/ClickHouse/pull/51148) ([Ilya Yatsishin](https://github.com/qoega)).
+* Revert "Added ability to implicitly use file/hdfs/s3 table functions in clickhouse-local" [#51149](https://github.com/ClickHouse/ClickHouse/pull/51149) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* More profile events for fs cache [#51161](https://github.com/ClickHouse/ClickHouse/pull/51161) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Unforget to pass callback to readBigAt() in ParallelReadBuffer [#51165](https://github.com/ClickHouse/ClickHouse/pull/51165) ([Michael Kolupaev](https://github.com/al13n321)).
+* Update README.md [#51179](https://github.com/ClickHouse/ClickHouse/pull/51179) ([Tyler Hannan](https://github.com/tylerhannan)).
+* Update exception message [#51187](https://github.com/ClickHouse/ClickHouse/pull/51187) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Split long test 02149_schema_inference_formats_with_schema into several tests to avoid timeout in debug [#51197](https://github.com/ClickHouse/ClickHouse/pull/51197) ([Kruglov Pavel](https://github.com/Avogar)).
+* Avoid initializing DateLUT from emptyArray function registration [#51199](https://github.com/ClickHouse/ClickHouse/pull/51199) ([Alexander Gololobov](https://github.com/davenger)).
+* Suppress check for covered parts in ZooKeeper [#51207](https://github.com/ClickHouse/ClickHouse/pull/51207) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* One more profile event for fs cache [#51223](https://github.com/ClickHouse/ClickHouse/pull/51223) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Typo: passowrd_sha256_hex --> password_sha256_hex [#51233](https://github.com/ClickHouse/ClickHouse/pull/51233) ([Robert Schulze](https://github.com/rschu1ze)).
+* Introduce settings enum field with auto-generated values list [#51237](https://github.com/ClickHouse/ClickHouse/pull/51237) ([Sergei Trifonov](https://github.com/serxa)).
+* Drop session if we fail to get Keeper API version [#51238](https://github.com/ClickHouse/ClickHouse/pull/51238) ([Alexander Gololobov](https://github.com/davenger)).
+* Revert "Fix a crash in s3 and s3Cluster functions" [#51239](https://github.com/ClickHouse/ClickHouse/pull/51239) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* fix flaky `AsyncLoader` destructor [#51245](https://github.com/ClickHouse/ClickHouse/pull/51245) ([Sergei Trifonov](https://github.com/serxa)).
+* Docs: little cleanup of configuration-files.md [#51249](https://github.com/ClickHouse/ClickHouse/pull/51249) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix a stupid bug on Replicated database recovery [#51252](https://github.com/ClickHouse/ClickHouse/pull/51252) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* FileCache: tryReserve() slight improvement [#51259](https://github.com/ClickHouse/ClickHouse/pull/51259) ([Igor Nikonov](https://github.com/devcrafter)).
+* Ugly hotfix for "terminate on uncaught exception" in WriteBufferFromOStream [#51265](https://github.com/ClickHouse/ClickHouse/pull/51265) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Avoid too many calls to Poco::Logger::get [#51266](https://github.com/ClickHouse/ClickHouse/pull/51266) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Update version_date.tsv and changelogs after v23.3.5.9-lts [#51269](https://github.com/ClickHouse/ClickHouse/pull/51269) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Better reporting of broken parts [#51270](https://github.com/ClickHouse/ClickHouse/pull/51270) ([Anton Popov](https://github.com/CurtizJ)).
+* Update ext-dict-functions.md [#51283](https://github.com/ClickHouse/ClickHouse/pull/51283) ([Mike Kot](https://github.com/myrrc)).
+* Disable table structure check for secondary queries from Replicated db [#51284](https://github.com/ClickHouse/ClickHouse/pull/51284) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Define Thrift version for parquet and use correct arrow version [#51285](https://github.com/ClickHouse/ClickHouse/pull/51285) ([Kruglov Pavel](https://github.com/Avogar)).
+* Restore Azure build on ARM [#51288](https://github.com/ClickHouse/ClickHouse/pull/51288) ([Robert Schulze](https://github.com/rschu1ze)).
+* Query Cache: Un-comment settings in server cfg [#51294](https://github.com/ClickHouse/ClickHouse/pull/51294) ([Robert Schulze](https://github.com/rschu1ze)).
+* Require more checks [#51295](https://github.com/ClickHouse/ClickHouse/pull/51295) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix metadata loading test [#51297](https://github.com/ClickHouse/ClickHouse/pull/51297) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Scratch the strange Python code [#51302](https://github.com/ClickHouse/ClickHouse/pull/51302) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add a test for [#47865](https://github.com/ClickHouse/ClickHouse/issues/47865) [#51306](https://github.com/ClickHouse/ClickHouse/pull/51306) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add a test for [#48894](https://github.com/ClickHouse/ClickHouse/issues/48894) [#51307](https://github.com/ClickHouse/ClickHouse/pull/51307) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add a test for [#48676](https://github.com/ClickHouse/ClickHouse/issues/48676) [#51308](https://github.com/ClickHouse/ClickHouse/pull/51308) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix long test `functions_bad_arguments` [#51310](https://github.com/ClickHouse/ClickHouse/pull/51310) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Unify merge predicate [#51344](https://github.com/ClickHouse/ClickHouse/pull/51344) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix using locks in ProcessList [#51348](https://github.com/ClickHouse/ClickHouse/pull/51348) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Add a test for [#42631](https://github.com/ClickHouse/ClickHouse/issues/42631) [#51353](https://github.com/ClickHouse/ClickHouse/pull/51353) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix performance tests due to warnings from jemalloc about Per-CPU arena disabled [#51362](https://github.com/ClickHouse/ClickHouse/pull/51362) ([Azat Khuzhin](https://github.com/azat)).
+* Fix "merge_truncate_long" test [#51369](https://github.com/ClickHouse/ClickHouse/pull/51369) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Increase timeout of Fast Test [#51372](https://github.com/ClickHouse/ClickHouse/pull/51372) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix bad tests for DNS [#51374](https://github.com/ClickHouse/ClickHouse/pull/51374) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Attempt to fix the `relax_too_many_parts` test [#51375](https://github.com/ClickHouse/ClickHouse/pull/51375) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix MySQL test in Debug mode [#51376](https://github.com/ClickHouse/ClickHouse/pull/51376) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix bad test `01018_Distributed__shard_num` [#51377](https://github.com/ClickHouse/ClickHouse/pull/51377) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix "logical error" in addressToLineWithInlines [#51379](https://github.com/ClickHouse/ClickHouse/pull/51379) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix test 01280_ttl_where_group_by [#51380](https://github.com/ClickHouse/ClickHouse/pull/51380) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Attempt to fix `test_ssl_cert_authentication` [#51384](https://github.com/ClickHouse/ClickHouse/pull/51384) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Revert "Merge pull request [#50951](https://github.com/ClickHouse/ClickHouse/issues/50951) from ZhiguoZh/20230607-toyear-fix" [#51390](https://github.com/ClickHouse/ClickHouse/pull/51390) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Two tests are twice longer in average with Analyzer and sometimes failing [#51391](https://github.com/ClickHouse/ClickHouse/pull/51391) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix 00899_long_attach_memory_limit [#51395](https://github.com/ClickHouse/ClickHouse/pull/51395) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix test 01293_optimize_final_force [#51396](https://github.com/ClickHouse/ClickHouse/pull/51396) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix test 02481_parquet_list_monotonically_increasing_offsets [#51397](https://github.com/ClickHouse/ClickHouse/pull/51397) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix test 02497_trace_events_stress_long [#51398](https://github.com/ClickHouse/ClickHouse/pull/51398) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix broken labeling for `manual approve` [#51405](https://github.com/ClickHouse/ClickHouse/pull/51405) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix parts lifetime in `MergeTreeTransaction` [#51407](https://github.com/ClickHouse/ClickHouse/pull/51407) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix flaky test test_skip_empty_files [#51409](https://github.com/ClickHouse/ClickHouse/pull/51409) ([Kruglov Pavel](https://github.com/Avogar)).
+* fix flacky test test_profile_events_s3 [#51412](https://github.com/ClickHouse/ClickHouse/pull/51412) ([Sema Checherinda](https://github.com/CheSema)).
+* Update README.md [#51413](https://github.com/ClickHouse/ClickHouse/pull/51413) ([Tyler Hannan](https://github.com/tylerhannan)).
+* Replace try/catch logic in hasTokenOrNull() by something more lightweight [#51425](https://github.com/ClickHouse/ClickHouse/pull/51425) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add retries to `tlsv1_3` tests [#51434](https://github.com/ClickHouse/ClickHouse/pull/51434) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Update exception message [#51440](https://github.com/ClickHouse/ClickHouse/pull/51440) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* fs cache: add check for intersecting ranges [#51444](https://github.com/ClickHouse/ClickHouse/pull/51444) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Slightly better code around packets for parallel replicas [#51451](https://github.com/ClickHouse/ClickHouse/pull/51451) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Update system_warnings test [#51453](https://github.com/ClickHouse/ClickHouse/pull/51453) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Many fixes [#51455](https://github.com/ClickHouse/ClickHouse/pull/51455) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix test 01605_adaptive_granularity_block_borders [#51457](https://github.com/ClickHouse/ClickHouse/pull/51457) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Try fix flaky 02497_storage_file_reader_selection [#51468](https://github.com/ClickHouse/ClickHouse/pull/51468) ([Kruglov Pavel](https://github.com/Avogar)).
+* Try making Keeper in `DatabaseReplicated` tests more stable [#51473](https://github.com/ClickHouse/ClickHouse/pull/51473) ([Antonio Andelic](https://github.com/antonio2368)).
+* Convert 02003_memory_limit_in_client from expect to sh test (to fix flakiness) [#51475](https://github.com/ClickHouse/ClickHouse/pull/51475) ([Azat Khuzhin](https://github.com/azat)).
+* Fix test_disk_over_web_server [#51476](https://github.com/ClickHouse/ClickHouse/pull/51476) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Delay shutdown of system and temporary databases [#51479](https://github.com/ClickHouse/ClickHouse/pull/51479) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix memory leakage in CompressionCodecDeflateQpl [#51480](https://github.com/ClickHouse/ClickHouse/pull/51480) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Increase retries in test_multiple_disks/test.py::test_start_stop_moves [#51482](https://github.com/ClickHouse/ClickHouse/pull/51482) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix race in BoundedReadBuffer [#51484](https://github.com/ClickHouse/ClickHouse/pull/51484) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky unit test [#51485](https://github.com/ClickHouse/ClickHouse/pull/51485) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky test `test_host_regexp_multiple_ptr_records` [#51506](https://github.com/ClickHouse/ClickHouse/pull/51506) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Add a comment [#51517](https://github.com/ClickHouse/ClickHouse/pull/51517) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Make `test_ssl_cert_authentication` similar to `test_tlvs1_3` [#51520](https://github.com/ClickHouse/ClickHouse/pull/51520) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fix duplicate storage set logical error. [#51521](https://github.com/ClickHouse/ClickHouse/pull/51521) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Update test_storage_postgresql/test.py::test_concurrent_queries [#51523](https://github.com/ClickHouse/ClickHouse/pull/51523) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix FATAL: query context is not detached from thread group [#51540](https://github.com/ClickHouse/ClickHouse/pull/51540) ([Igor Nikonov](https://github.com/devcrafter)).
+* Update version_date.tsv and changelogs after v23.3.6.7-lts [#51548](https://github.com/ClickHouse/ClickHouse/pull/51548) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Decoupled commits from [#51180](https://github.com/ClickHouse/ClickHouse/issues/51180) for backports [#51561](https://github.com/ClickHouse/ClickHouse/pull/51561) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Try to fix deadlock in ZooKeeper client [#51563](https://github.com/ClickHouse/ClickHouse/pull/51563) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Retry chroot creation in ZK before stateless tests [#51585](https://github.com/ClickHouse/ClickHouse/pull/51585) ([Antonio Andelic](https://github.com/antonio2368)).
+* use timeout instead trap in 01443_merge_truncate_long.sh [#51593](https://github.com/ClickHouse/ClickHouse/pull/51593) ([Sema Checherinda](https://github.com/CheSema)).
+* Update version_date.tsv and changelogs after v23.5.4.25-stable [#51604](https://github.com/ClickHouse/ClickHouse/pull/51604) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix MergeTreeMarksLoader segfaulting if marks file is longer than expected [#51636](https://github.com/ClickHouse/ClickHouse/pull/51636) ([Michael Kolupaev](https://github.com/al13n321)).
+* Update version_date.tsv and changelogs after v23.4.5.22-stable [#51638](https://github.com/ClickHouse/ClickHouse/pull/51638) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update version_date.tsv and changelogs after v23.3.7.5-lts [#51639](https://github.com/ClickHouse/ClickHouse/pull/51639) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Update parts.md [#51643](https://github.com/ClickHouse/ClickHouse/pull/51643) ([Ramazan Polat](https://github.com/ramazanpolat)).
+
diff --git a/docs/changelogs/v23.6.2.18-stable.md b/docs/changelogs/v23.6.2.18-stable.md
new file mode 100644
index 00000000000..1f872a190ba
--- /dev/null
+++ b/docs/changelogs/v23.6.2.18-stable.md
@@ -0,0 +1,25 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.6.2.18-stable (89f39a7ccfe) FIXME as compared to v23.6.1.1524-stable (d1c7e13d088)
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#51888](https://github.com/ClickHouse/ClickHouse/issues/51888): Update cargo dependencies. [#51721](https://github.com/ClickHouse/ClickHouse/pull/51721) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix reading from empty column in `parseSipHashKey` [#51804](https://github.com/ClickHouse/ClickHouse/pull/51804) ([Nikita Taranov](https://github.com/nickitat)).
+* Allow parametric UDFs [#51964](https://github.com/ClickHouse/ClickHouse/pull/51964) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Remove the usage of Analyzer setting in the client [#51578](https://github.com/ClickHouse/ClickHouse/pull/51578) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix 02116_tuple_element with Analyzer [#51669](https://github.com/ClickHouse/ClickHouse/pull/51669) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix SQLLogic docker images [#51719](https://github.com/ClickHouse/ClickHouse/pull/51719) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix source image for sqllogic [#51728](https://github.com/ClickHouse/ClickHouse/pull/51728) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Pin for docker-ce [#51743](https://github.com/ClickHouse/ClickHouse/pull/51743) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
diff --git a/docs/en/development/build-cross-riscv.md b/docs/en/development/build-cross-riscv.md
index e3550a046c7..c21353f7f73 100644
--- a/docs/en/development/build-cross-riscv.md
+++ b/docs/en/development/build-cross-riscv.md
@@ -23,7 +23,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
``` bash
cd ClickHouse
mkdir build-riscv64
-CC=clang-16 CXX=clang++-16 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DUSE_UNWIND=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF
+CC=clang-16 CXX=clang++-16 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF
ninja -C build-riscv64
```
diff --git a/docs/en/development/build.md b/docs/en/development/build.md
index 83a4550df88..e3749608bbc 100644
--- a/docs/en/development/build.md
+++ b/docs/en/development/build.md
@@ -11,7 +11,8 @@ Supported platforms:
- x86_64
- AArch64
-- Power9 (experimental)
+- PowerPC 64 LE (experimental)
+- RISC-V 64 (experimental)
## Building on Ubuntu
@@ -42,7 +43,7 @@ sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
For other Linux distribution - check the availability of LLVM's [prebuild packages](https://releases.llvm.org/download.html).
-As of April 2023, any version of Clang >= 15 will work.
+As of April 2023, clang-16 or higher will work.
GCC as a compiler is not supported.
To build with a specific Clang version:
@@ -86,8 +87,8 @@ The build requires the following components:
- Git (used to checkout the sources, not needed for the build)
- CMake 3.20 or newer
-- Compiler: Clang 15 or newer
-- Linker: lld 15 or newer
+- Compiler: clang-16 or newer
+- Linker: lld-16 or newer
- Ninja
- Yasm
- Gawk
diff --git a/docs/en/development/building_and_benchmarking_deflate_qpl.md b/docs/en/development/building_and_benchmarking_deflate_qpl.md
index 0501c1cbdcb..4e01b41ab3c 100644
--- a/docs/en/development/building_and_benchmarking_deflate_qpl.md
+++ b/docs/en/development/building_and_benchmarking_deflate_qpl.md
@@ -7,12 +7,8 @@ description: How to build Clickhouse and run benchmark with DEFLATE_QPL Codec
# Build Clickhouse with DEFLATE_QPL
-- Make sure your target machine meet the QPL required [prerequisites](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#prerequisites)
-- Pass the following flag to CMake when building ClickHouse:
-
-``` bash
-cmake -DENABLE_QPL=1 ..
-```
+- Make sure your host machine meet the QPL required [prerequisites](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#prerequisites)
+- deflate_qpl is enabled by default during cmake build. In case you accidentally change it, please double-check build flag: ENABLE_QPL=1
- For generic requirements, please refer to Clickhouse generic [build instructions](/docs/en/development/build.md)
diff --git a/docs/en/engines/table-engines/integrations/ExternalDistributed.md b/docs/en/engines/table-engines/integrations/ExternalDistributed.md
index 3fb3fe88b55..d995c34e00a 100644
--- a/docs/en/engines/table-engines/integrations/ExternalDistributed.md
+++ b/docs/en/engines/table-engines/integrations/ExternalDistributed.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/ExternalDistributed
-sidebar_position: 12
+sidebar_position: 55
sidebar_label: ExternalDistributed
title: ExternalDistributed
---
diff --git a/docs/en/engines/table-engines/integrations/azureBlobStorage.md b/docs/en/engines/table-engines/integrations/azureBlobStorage.md
index 14fbf0c068e..60e448377d0 100644
--- a/docs/en/engines/table-engines/integrations/azureBlobStorage.md
+++ b/docs/en/engines/table-engines/integrations/azureBlobStorage.md
@@ -1,5 +1,6 @@
---
slug: /en/engines/table-engines/integrations/azureBlobStorage
+sidebar_position: 10
sidebar_label: Azure Blob Storage
---
@@ -29,8 +30,8 @@ CREATE TABLE azure_blob_storage_table (name String, value UInt32)
**Example**
``` sql
-CREATE TABLE test_table (key UInt64, data String)
- ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/;',
+CREATE TABLE test_table (key UInt64, data String)
+ ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/;',
'test_container', 'test_table', 'CSV');
INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');
diff --git a/docs/en/engines/table-engines/integrations/deltalake.md b/docs/en/engines/table-engines/integrations/deltalake.md
index 3e2e177e28f..b562e9d7fe6 100644
--- a/docs/en/engines/table-engines/integrations/deltalake.md
+++ b/docs/en/engines/table-engines/integrations/deltalake.md
@@ -1,5 +1,6 @@
---
slug: /en/engines/table-engines/integrations/deltalake
+sidebar_position: 40
sidebar_label: DeltaLake
---
diff --git a/docs/en/engines/table-engines/integrations/embedded-rocksdb.md b/docs/en/engines/table-engines/integrations/embedded-rocksdb.md
index 6664b6a4613..2de981d33b7 100644
--- a/docs/en/engines/table-engines/integrations/embedded-rocksdb.md
+++ b/docs/en/engines/table-engines/integrations/embedded-rocksdb.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/embedded-rocksdb
-sidebar_position: 9
+sidebar_position: 50
sidebar_label: EmbeddedRocksDB
---
@@ -99,7 +99,7 @@ INSERT INTO test VALUES ('some key', 1, 'value', 3.2);
### Deletes
-Rows can be deleted using `DELETE` query or `TRUNCATE`.
+Rows can be deleted using `DELETE` query or `TRUNCATE`.
```sql
DELETE FROM test WHERE key LIKE 'some%' AND v1 > 1;
diff --git a/docs/en/engines/table-engines/integrations/hdfs.md b/docs/en/engines/table-engines/integrations/hdfs.md
index ff6e3a3bd98..c677123a8d0 100644
--- a/docs/en/engines/table-engines/integrations/hdfs.md
+++ b/docs/en/engines/table-engines/integrations/hdfs.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/hdfs
-sidebar_position: 6
+sidebar_position: 80
sidebar_label: HDFS
---
@@ -63,7 +63,7 @@ SELECT * FROM hdfs_engine_table LIMIT 2
- `ALTER` and `SELECT...SAMPLE` operations.
- Indexes.
- [Zero-copy](../../../operations/storing-data.md#zero-copy) replication is possible, but not recommended.
-
+
:::note Zero-copy replication is not ready for production
Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use.
:::
diff --git a/docs/en/engines/table-engines/integrations/hive.md b/docs/en/engines/table-engines/integrations/hive.md
index 5d10e417ae3..48867394418 100644
--- a/docs/en/engines/table-engines/integrations/hive.md
+++ b/docs/en/engines/table-engines/integrations/hive.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/hive
-sidebar_position: 4
+sidebar_position: 84
sidebar_label: Hive
---
diff --git a/docs/en/engines/table-engines/integrations/hudi.md b/docs/en/engines/table-engines/integrations/hudi.md
index a11e915aa3d..c60618af289 100644
--- a/docs/en/engines/table-engines/integrations/hudi.md
+++ b/docs/en/engines/table-engines/integrations/hudi.md
@@ -1,5 +1,6 @@
---
slug: /en/engines/table-engines/integrations/hudi
+sidebar_position: 86
sidebar_label: Hudi
---
diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md
index 77cefc9283d..9d6395f73ac 100644
--- a/docs/en/engines/table-engines/integrations/iceberg.md
+++ b/docs/en/engines/table-engines/integrations/iceberg.md
@@ -1,5 +1,6 @@
---
slug: /en/engines/table-engines/integrations/iceberg
+sidebar_position: 90
sidebar_label: Iceberg
---
diff --git a/docs/en/engines/table-engines/integrations/jdbc.md b/docs/en/engines/table-engines/integrations/jdbc.md
index 99f851dcf3e..a4a1e2a31ae 100644
--- a/docs/en/engines/table-engines/integrations/jdbc.md
+++ b/docs/en/engines/table-engines/integrations/jdbc.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/jdbc
-sidebar_position: 3
+sidebar_position: 100
sidebar_label: JDBC
---
diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md
index ccfca4c1f1f..b81d5624c1a 100644
--- a/docs/en/engines/table-engines/integrations/kafka.md
+++ b/docs/en/engines/table-engines/integrations/kafka.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/kafka
-sidebar_position: 8
+sidebar_position: 110
sidebar_label: Kafka
---
diff --git a/docs/en/engines/table-engines/integrations/materialized-postgresql.md b/docs/en/engines/table-engines/integrations/materialized-postgresql.md
index e112ca3bbb1..bccafd67c2c 100644
--- a/docs/en/engines/table-engines/integrations/materialized-postgresql.md
+++ b/docs/en/engines/table-engines/integrations/materialized-postgresql.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/materialized-postgresql
-sidebar_position: 12
+sidebar_position: 130
sidebar_label: MaterializedPostgreSQL
title: MaterializedPostgreSQL
---
diff --git a/docs/en/engines/table-engines/integrations/mongodb.md b/docs/en/engines/table-engines/integrations/mongodb.md
index a647ac9993f..f87e8da8b5b 100644
--- a/docs/en/engines/table-engines/integrations/mongodb.md
+++ b/docs/en/engines/table-engines/integrations/mongodb.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/mongodb
-sidebar_position: 5
+sidebar_position: 135
sidebar_label: MongoDB
---
@@ -33,6 +33,15 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name
- `options` — MongoDB connection string options (optional parameter).
+:::tip
+If you are using the MongoDB Atlas cloud offering please add these options:
+
+```
+'connectTimeoutMS=10000&ssl=true&authSource=admin'
+```
+
+:::
+
## Usage Example {#usage-example}
Create a table in ClickHouse which allows to read data from MongoDB collection:
diff --git a/docs/en/engines/table-engines/integrations/mysql.md b/docs/en/engines/table-engines/integrations/mysql.md
index 6ff6221c877..e50ed8caedd 100644
--- a/docs/en/engines/table-engines/integrations/mysql.md
+++ b/docs/en/engines/table-engines/integrations/mysql.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/mysql
-sidebar_position: 4
+sidebar_position: 138
sidebar_label: MySQL
---
@@ -35,6 +35,10 @@ The table structure can differ from the original MySQL table structure:
- Column types may differ from those in the original MySQL table. ClickHouse tries to [cast](../../../engines/database-engines/mysql.md#data_types-support) values to the ClickHouse data types.
- The [external_table_functions_use_nulls](../../../operations/settings/settings.md#external-table-functions-use-nulls) setting defines how to handle Nullable columns. Default value: 1. If 0, the table function does not make Nullable columns and inserts default values instead of nulls. This is also applicable for NULL values inside arrays.
+:::note
+The MySQL Table Engine is currently not available on the ClickHouse builds for MacOS ([issue](https://github.com/ClickHouse/ClickHouse/issues/21191))
+:::
+
**Engine Parameters**
- `host:port` — MySQL server address.
diff --git a/docs/en/engines/table-engines/integrations/nats.md b/docs/en/engines/table-engines/integrations/nats.md
index a82d74e0d95..25f442e5ce7 100644
--- a/docs/en/engines/table-engines/integrations/nats.md
+++ b/docs/en/engines/table-engines/integrations/nats.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/nats
-sidebar_position: 14
+sidebar_position: 140
sidebar_label: NATS
---
@@ -83,12 +83,12 @@ You can select one of the subjects the table reads from and publish your data th
CREATE TABLE queue (
key UInt64,
value UInt64
- ) ENGINE = NATS
+ ) ENGINE = NATS
SETTINGS nats_url = 'localhost:4444',
nats_subjects = 'subject1,subject2',
nats_format = 'JSONEachRow';
- INSERT INTO queue
+ INSERT INTO queue
SETTINGS stream_like_engine_insert_queue = 'subject2'
VALUES (1, 1);
```
@@ -102,7 +102,7 @@ Example:
key UInt64,
value UInt64,
date DateTime
- ) ENGINE = NATS
+ ) ENGINE = NATS
SETTINGS nats_url = 'localhost:4444',
nats_subjects = 'subject1',
nats_format = 'JSONEachRow',
@@ -137,7 +137,7 @@ Example:
CREATE TABLE queue (
key UInt64,
value UInt64
- ) ENGINE = NATS
+ ) ENGINE = NATS
SETTINGS nats_url = 'localhost:4444',
nats_subjects = 'subject1',
nats_format = 'JSONEachRow',
diff --git a/docs/en/engines/table-engines/integrations/odbc.md b/docs/en/engines/table-engines/integrations/odbc.md
index 37e08dc1420..71085feb626 100644
--- a/docs/en/engines/table-engines/integrations/odbc.md
+++ b/docs/en/engines/table-engines/integrations/odbc.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/odbc
-sidebar_position: 2
+sidebar_position: 150
sidebar_label: ODBC
---
@@ -54,7 +54,7 @@ $ sudo mysql
``` sql
mysql> CREATE USER 'clickhouse'@'localhost' IDENTIFIED BY 'clickhouse';
-mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'clickhouse' WITH GRANT OPTION;
+mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'localhost' WITH GRANT OPTION;
```
Then configure the connection in `/etc/odbc.ini`.
@@ -66,7 +66,7 @@ DRIVER = /usr/local/lib/libmyodbc5w.so
SERVER = 127.0.0.1
PORT = 3306
DATABASE = test
-USERNAME = clickhouse
+USER = clickhouse
PASSWORD = clickhouse
```
@@ -83,6 +83,9 @@ $ isql -v mysqlconn
Table in MySQL:
``` text
+mysql> CREATE DATABASE test;
+Query OK, 1 row affected (0,01 sec)
+
mysql> CREATE TABLE `test`.`test` (
-> `int_id` INT NOT NULL AUTO_INCREMENT,
-> `int_nullable` INT NULL DEFAULT NULL,
@@ -91,10 +94,10 @@ mysql> CREATE TABLE `test`.`test` (
-> PRIMARY KEY (`int_id`));
Query OK, 0 rows affected (0,09 sec)
-mysql> insert into test (`int_id`, `float`) VALUES (1,2);
+mysql> insert into test.test (`int_id`, `float`) VALUES (1,2);
Query OK, 1 row affected (0,00 sec)
-mysql> select * from test;
+mysql> select * from test.test;
+------+----------+-----+----------+
| int_id | int_nullable | float | float_nullable |
+------+----------+-----+----------+
diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md
index 8eab6fdb421..aa3dc855537 100644
--- a/docs/en/engines/table-engines/integrations/postgresql.md
+++ b/docs/en/engines/table-engines/integrations/postgresql.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/postgresql
-sidebar_position: 11
+sidebar_position: 160
sidebar_label: PostgreSQL
---
diff --git a/docs/en/engines/table-engines/integrations/rabbitmq.md b/docs/en/engines/table-engines/integrations/rabbitmq.md
index 7620cd22767..3fd5a130173 100644
--- a/docs/en/engines/table-engines/integrations/rabbitmq.md
+++ b/docs/en/engines/table-engines/integrations/rabbitmq.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/rabbitmq
-sidebar_position: 10
+sidebar_position: 170
sidebar_label: RabbitMQ
---
diff --git a/docs/en/engines/table-engines/integrations/redis.md b/docs/en/engines/table-engines/integrations/redis.md
index c1e88e7e92e..8086a6503b8 100644
--- a/docs/en/engines/table-engines/integrations/redis.md
+++ b/docs/en/engines/table-engines/integrations/redis.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/redis
-sidebar_position: 43
+sidebar_position: 175
sidebar_label: Redis
---
@@ -44,11 +44,12 @@ Create a table in ClickHouse which allows to read data from Redis:
``` sql
CREATE TABLE redis_table
(
- `k` String,
- `m` String,
- `n` UInt32
+ `key` String,
+ `v1` UInt32,
+ `v2` String,
+ `v3` Float32
)
-ENGINE = Redis('redis1:6379') PRIMARY KEY(k);
+ENGINE = Redis('redis1:6379') PRIMARY KEY(key);
```
Insert:
@@ -111,9 +112,16 @@ Flush Redis db asynchronously. Also `Truncate` support SYNC mode.
TRUNCATE TABLE redis_table SYNC;
```
+Join:
+
+Join with other tables.
+
+```
+SELECT * FROM redis_table JOIN merge_tree_table ON merge_tree_table.key=redis_table.key;
+```
## Limitations {#limitations}
Redis engine also supports scanning queries, such as `where k > xx`, but it has some limitations:
-1. Scanning query may produce some duplicated keys in a very rare case when it is rehashing. See details in [Redis Scan](https://github.com/redis/redis/blob/e4d183afd33e0b2e6e8d1c79a832f678a04a7886/src/dict.c#L1186-L1269)
+1. Scanning query may produce some duplicated keys in a very rare case when it is rehashing. See details in [Redis Scan](https://github.com/redis/redis/blob/e4d183afd33e0b2e6e8d1c79a832f678a04a7886/src/dict.c#L1186-L1269).
2. During the scanning, keys could be created and deleted, so the resulting dataset can not represent a valid point in time.
diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md
index 10a72394b57..051945538b2 100644
--- a/docs/en/engines/table-engines/integrations/s3.md
+++ b/docs/en/engines/table-engines/integrations/s3.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/s3
-sidebar_position: 7
+sidebar_position: 180
sidebar_label: S3
---
@@ -8,30 +8,7 @@ sidebar_label: S3
This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ecosystem. This engine is similar to the [HDFS](../../../engines/table-engines/special/file.md#table_engines-hdfs) engine, but provides S3-specific features.
-## Create Table {#creating-a-table}
-
-``` sql
-CREATE TABLE s3_engine_table (name String, value UInt32)
- ENGINE = S3(path [, NOSIGN | aws_access_key_id, aws_secret_access_key,] format, [compression])
- [PARTITION BY expr]
- [SETTINGS ...]
-```
-
-**Engine parameters**
-
-- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [below](#wildcards-in-path).
-- `NOSIGN` - If this keyword is provided in place of credentials, all the requests will not be signed.
-- `format` — The [format](../../../interfaces/formats.md#formats) of the file.
-- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3).
-- `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will auto-detect compression by file extension.
-
-### PARTITION BY
-
-`PARTITION BY` — Optional. In most cases you don't need a partition key, and if it is needed you generally don't need a partition key more granular than by month. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead, make client identifier or name the first column in the ORDER BY expression).
-
-For partitioning by month, use the `toYYYYMM(date_column)` expression, where `date_column` is a column with a date of the type [Date](/docs/en/sql-reference/data-types/date.md). The partition names here have the `"YYYYMM"` format.
-
-**Example**
+## Example
``` sql
CREATE TABLE s3_engine_table (name String, value UInt32)
@@ -49,6 +26,120 @@ SELECT * FROM s3_engine_table LIMIT 2;
│ two │ 2 │
└──────┴───────┘
```
+## Create Table {#creating-a-table}
+
+``` sql
+CREATE TABLE s3_engine_table (name String, value UInt32)
+ ENGINE = S3(path [, NOSIGN | aws_access_key_id, aws_secret_access_key,] format, [compression])
+ [PARTITION BY expr]
+ [SETTINGS ...]
+```
+
+### Engine parameters
+
+- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [below](#wildcards-in-path).
+- `NOSIGN` - If this keyword is provided in place of credentials, all the requests will not be signed.
+- `format` — The [format](../../../interfaces/formats.md#formats) of the file.
+- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3).
+- `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will auto-detect compression by file extension.
+
+### PARTITION BY
+
+`PARTITION BY` — Optional. In most cases you don't need a partition key, and if it is needed you generally don't need a partition key more granular than by month. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead, make client identifier or name the first column in the ORDER BY expression).
+
+For partitioning by month, use the `toYYYYMM(date_column)` expression, where `date_column` is a column with a date of the type [Date](/docs/en/sql-reference/data-types/date.md). The partition names here have the `"YYYYMM"` format.
+
+### Querying partitioned data
+
+This example uses the [docker compose recipe](https://github.com/ClickHouse/examples/tree/5fdc6ff72f4e5137e23ea075c88d3f44b0202490/docker-compose-recipes/recipes/ch-and-minio-S3), which integrates ClickHouse and MinIO. You should be able to reproduce the same queries using S3 by replacing the endpoint and authentication values.
+
+Notice that the S3 endpoint in the `ENGINE` configuration uses the parameter token `{_partition_id}` as part of the S3 object (filename), and that the SELECT queries select against those resulting object names (e.g., `test_3.csv`).
+
+:::note
+As shown in the example, querying from S3 tables that are partitioned is
+not directly supported at this time, but can be accomplished by querying the individual partitions
+using the S3 table function.
+
+The primary use-case for writing
+partitioned data in S3 is to enable transferring that data into another
+ClickHouse system (for example, moving from on-prem systems to ClickHouse
+Cloud). Because ClickHouse datasets are often very large, and network
+reliability is sometimes imperfect it makes sense to transfer datasets
+in subsets, hence partitioned writes.
+:::
+
+#### Create the table
+```sql
+CREATE TABLE p
+(
+ `column1` UInt32,
+ `column2` UInt32,
+ `column3` UInt32
+)
+ENGINE = S3(
+# highlight-next-line
+ 'http://minio:10000/clickhouse//test_{_partition_id}.csv',
+ 'minioadmin',
+ 'minioadminpassword',
+ 'CSV')
+PARTITION BY column3
+```
+
+#### Insert data
+```sql
+insert into p values (1, 2, 3), (3, 2, 1), (78, 43, 45)
+```
+
+#### Select from partition 3
+
+:::tip
+This query uses the s3 table function
+:::
+
+```sql
+SELECT *
+FROM s3('http://minio:10000/clickhouse//test_3.csv', 'minioadmin', 'minioadminpassword', 'CSV')
+```
+```response
+┌─c1─┬─c2─┬─c3─┐
+│ 1 │ 2 │ 3 │
+└────┴────┴────┘
+```
+
+#### Select from partition 1
+```sql
+SELECT *
+FROM s3('http://minio:10000/clickhouse//test_1.csv', 'minioadmin', 'minioadminpassword', 'CSV')
+```
+```response
+┌─c1─┬─c2─┬─c3─┐
+│ 3 │ 2 │ 1 │
+└────┴────┴────┘
+```
+
+#### Select from partition 45
+```sql
+SELECT *
+FROM s3('http://minio:10000/clickhouse//test_45.csv', 'minioadmin', 'minioadminpassword', 'CSV')
+```
+```response
+┌─c1─┬─c2─┬─c3─┐
+│ 78 │ 43 │ 45 │
+└────┴────┴────┘
+```
+
+#### Limitation
+
+You may naturally try to `Select * from p`, but as noted above, this query will fail; use the preceding query.
+
+```sql
+SELECT * FROM p
+```
+```response
+Received exception from server (version 23.4.1):
+Code: 48. DB::Exception: Received from localhost:9000. DB::Exception: Reading from a partitioned S3 storage is not implemented yet. (NOT_IMPLEMENTED)
+```
+
## Virtual columns {#virtual-columns}
- `_path` — Path to the file.
diff --git a/docs/en/engines/table-engines/integrations/sqlite.md b/docs/en/engines/table-engines/integrations/sqlite.md
index 20597d37a87..c67f863d390 100644
--- a/docs/en/engines/table-engines/integrations/sqlite.md
+++ b/docs/en/engines/table-engines/integrations/sqlite.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-engines/integrations/sqlite
-sidebar_position: 7
+sidebar_position: 185
sidebar_label: SQLite
---
diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md
index 42454af6feb..4f506126682 100644
--- a/docs/en/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md
@@ -37,8 +37,8 @@ The [Merge](/docs/en/engines/table-engines/special/merge.md/#merge) engine does
``` sql
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
(
- name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1],
- name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2],
+ name1 [type1] [DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL expr1] [TTL expr1] [CODEC(codec1)] [[NOT] NULL|PRIMARY KEY],
+ name2 [type2] [DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL expr2] [TTL expr2] [CODEC(codec2)] [[NOT] NULL|PRIMARY KEY],
...
INDEX index_name1 expr1 TYPE type1(...) [GRANULARITY value1],
INDEX index_name2 expr2 TYPE type2(...) [GRANULARITY value2],
@@ -439,41 +439,41 @@ Syntax: `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions,
- `number_of_hash_functions` — The number of hash functions used in the Bloom filter.
- `random_seed` — The seed for Bloom filter hash functions.
-Users can create [UDF](/docs/en/sql-reference/statements/create/function.md) to estimate the parameters set of `ngrambf_v1`. Query statements are as follows:
+Users can create [UDF](/docs/en/sql-reference/statements/create/function.md) to estimate the parameters set of `ngrambf_v1`. Query statements are as follows:
```sql
-CREATE FUNCTION bfEstimateFunctions [ON CLUSTER cluster]
-AS
-(total_nubmer_of_all_grams, size_of_bloom_filter_in_bits) -> round((size_of_bloom_filter_in_bits / total_nubmer_of_all_grams) * log(2));
-
-CREATE FUNCTION bfEstimateBmSize [ON CLUSTER cluster]
-AS
-(total_nubmer_of_all_grams, probability_of_false_positives) -> ceil((total_nubmer_of_all_grams * log(probability_of_false_positives)) / log(1 / pow(2, log(2))));
-
-CREATE FUNCTION bfEstimateFalsePositive [ON CLUSTER cluster]
-AS
-(total_nubmer_of_all_grams, number_of_hash_functions, size_of_bloom_filter_in_bytes) -> pow(1 - exp(-number_of_hash_functions/ (size_of_bloom_filter_in_bytes / total_nubmer_of_all_grams)), number_of_hash_functions);
-
-CREATE FUNCTION bfEstimateGramNumber [ON CLUSTER cluster]
-AS
+CREATE FUNCTION bfEstimateFunctions [ON CLUSTER cluster]
+AS
+(total_nubmer_of_all_grams, size_of_bloom_filter_in_bits) -> round((size_of_bloom_filter_in_bits / total_nubmer_of_all_grams) * log(2));
+
+CREATE FUNCTION bfEstimateBmSize [ON CLUSTER cluster]
+AS
+(total_nubmer_of_all_grams, probability_of_false_positives) -> ceil((total_nubmer_of_all_grams * log(probability_of_false_positives)) / log(1 / pow(2, log(2))));
+
+CREATE FUNCTION bfEstimateFalsePositive [ON CLUSTER cluster]
+AS
+(total_nubmer_of_all_grams, number_of_hash_functions, size_of_bloom_filter_in_bytes) -> pow(1 - exp(-number_of_hash_functions/ (size_of_bloom_filter_in_bytes / total_nubmer_of_all_grams)), number_of_hash_functions);
+
+CREATE FUNCTION bfEstimateGramNumber [ON CLUSTER cluster]
+AS
(number_of_hash_functions, probability_of_false_positives, size_of_bloom_filter_in_bytes) -> ceil(size_of_bloom_filter_in_bytes / (-number_of_hash_functions / log(1 - exp(log(probability_of_false_positives) / number_of_hash_functions))))
-```
+```
To use those functions,we need to specify two parameter at least.
-For example, if there 4300 ngrams in the granule and we expect false positives to be less than 0.0001. The other parameters can be estimated by executing following queries:
-
+For example, if there 4300 ngrams in the granule and we expect false positives to be less than 0.0001. The other parameters can be estimated by executing following queries:
+
```sql
--- estimate number of bits in the filter
-SELECT bfEstimateBmSize(4300, 0.0001) / 8 as size_of_bloom_filter_in_bytes;
+SELECT bfEstimateBmSize(4300, 0.0001) / 8 as size_of_bloom_filter_in_bytes;
┌─size_of_bloom_filter_in_bytes─┐
│ 10304 │
└───────────────────────────────┘
-
+
--- estimate number of hash functions
SELECT bfEstimateFunctions(4300, bfEstimateBmSize(4300, 0.0001)) as number_of_hash_functions
-
+
┌─number_of_hash_functions─┐
│ 13 │
└──────────────────────────┘
@@ -756,6 +756,17 @@ If you perform the `SELECT` query between merges, you may get expired data. To a
- [ttl_only_drop_parts](/docs/en/operations/settings/settings.md/#ttl_only_drop_parts) setting
+## Disk types
+
+In addition to local block devices, ClickHouse supports these storage types:
+- [`s3` for S3 and MinIO](#table_engine-mergetree-s3)
+- [`gcs` for GCS](/docs/en/integrations/data-ingestion/gcs/index.md/#creating-a-disk)
+- [`blob_storage_disk` for Azure Blob Storage](#table_engine-mergetree-azure-blob-storage)
+- [`hdfs` for HDFS](#hdfs-storage)
+- [`web` for read-only from web](#web-storage)
+- [`cache` for local caching](/docs/en/operations/storing-data.md/#using-local-cache)
+- [`s3_plain` for backups to S3](/docs/en/operations/backup#backuprestore-using-an-s3-disk)
+
## Using Multiple Block Devices for Data Storage {#table_engine-mergetree-multiple-volumes}
### Introduction {#introduction}
@@ -936,7 +947,16 @@ configuration files; all the settings are in the CREATE/ATTACH query.
The example uses `type=web`, but any disk type can be configured as dynamic, even Local disk. Local disks require a path argument to be inside the server config parameter `custom_local_disks_base_directory`, which has no default, so set that also when using local disk.
:::
+#### Example dynamic web storage
+
+:::tip
+A [demo dataset](https://github.com/ClickHouse/web-tables-demo) is hosted in GitHub. To prepare your own tables for web storage see the tool [clickhouse-static-files-uploader](/docs/en/operations/storing-data.md/#storing-data-on-webserver)
+:::
+
+In this `ATTACH TABLE` query the `UUID` provided matches the directory name of the data, and the endpoint is the URL for the raw GitHub content.
+
```sql
+# highlight-next-line
ATTACH TABLE uk_price_paid UUID 'cf712b4f-2ca8-435c-ac23-c4393efe52f7'
(
price UInt32,
@@ -971,7 +991,7 @@ use a local disk to cache data from a table stored at a URL. Neither the cache d
nor the web storage is configured in the ClickHouse configuration files; both are
configured in the CREATE/ATTACH query settings.
-In the settings highlighted below notice that the disk of `type=web` is nested within
+In the settings highlighted below notice that the disk of `type=web` is nested within
the disk of `type=cache`.
```sql
@@ -1238,6 +1258,93 @@ Examples of working configurations can be found in integration tests directory (
Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use.
:::
+## HDFS storage {#hdfs-storage}
+
+In this sample configuration:
+- the disk is of type `hdfs`
+- the data is hosted at `hdfs://hdfs1:9000/clickhouse/`
+
+```xml
+
+
+
+
+ hdfs
+ hdfs://hdfs1:9000/clickhouse/
+ true
+
+
+ local
+ /
+
+
+
+
+
+
+ hdfs
+
+
+ hdd
+
+
+
+
+
+
+```
+
+## Web storage (read-only) {#web-storage}
+
+Web storage can be used for read-only purposes. An example use is for hosting sample
+data, or for migrating data.
+
+:::tip
+Storage can also be configured temporarily within a query, if a web dataset is not expected
+to be used routinely, see [dynamic storage](#dynamic-storage) and skip editing the
+configuration file.
+:::
+
+In this sample configuration:
+- the disk is of type `web`
+- the data is hosted at `http://nginx:80/test1/`
+- a cache on local storage is used
+
+```xml
+
+
+
+
+ web
+ http://nginx:80/test1/
+
+
+ cache
+ web
+ cached_web_cache/
+ 100000000
+
+
+
+
+
+
+ web
+
+
+
+
+
+
+ cached_web
+
+
+
+
+
+
+```
+
## Virtual Columns {#virtual-columns}
- `_part` — Name of a part.
diff --git a/docs/en/engines/table-engines/special/url.md b/docs/en/engines/table-engines/special/url.md
index 26d4975954f..f556df0a088 100644
--- a/docs/en/engines/table-engines/special/url.md
+++ b/docs/en/engines/table-engines/special/url.md
@@ -106,3 +106,4 @@ For partitioning by month, use the `toYYYYMM(date_column)` expression, where `da
## Storage Settings {#storage-settings}
- [engine_url_skip_empty_files](/docs/en/operations/settings/settings.md#engine_url_skip_empty_files) - allows to skip empty files while reading. Disabled by default.
+- [disable_url_encoding](/docs/en/operations/settings/settings.md#disable_url_encoding) -allows to disable decoding/encoding path in uri. Disabled by default.
diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md
index d44dc861888..d2e7ab30478 100644
--- a/docs/en/getting-started/install.md
+++ b/docs/en/getting-started/install.md
@@ -378,6 +378,10 @@ request](https://github.com/ClickHouse/ClickHouse/commits/master) and find CI ch
https://s3.amazonaws.com/clickhouse/builds/PRs/.../.../binary_aarch64_v80compat/clickhouse". You can then click the link to download the
build.
+### macOS-only: Install with Homebrew
+
+To install ClickHouse using the popular `brew` package manager, follow the instructions listed in the [ClickHouse Homebrew tap](https://github.com/ClickHouse/homebrew-clickhouse).
+
## Launch {#launch}
To start the server as a daemon, run:
diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index 5fd2b82375f..15f9d1f47bf 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -76,6 +76,7 @@ The supported formats are:
| [RowBinary](#rowbinary) | ✔ | ✔ |
| [RowBinaryWithNames](#rowbinarywithnamesandtypes) | ✔ | ✔ |
| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ |
+| [RowBinaryWithDefaults](#rowbinarywithdefaults) | ✔ | ✔ |
| [Native](#native) | ✔ | ✔ |
| [Null](#null) | ✗ | ✔ |
| [XML](#xml) | ✗ | ✔ |
@@ -471,6 +472,8 @@ The CSV format supports the output of totals and extremes the same way as `TabSe
- [input_format_csv_skip_trailing_empty_lines](/docs/en/operations/settings/settings-formats.md/#input_format_csv_skip_trailing_empty_lines) - skip trailing empty lines at the end of data. Default value - `false`.
- [input_format_csv_trim_whitespaces](/docs/en/operations/settings/settings-formats.md/#input_format_csv_trim_whitespaces) - trim spaces and tabs in non-quoted CSV strings. Default value - `true`.
- [input_format_csv_allow_whitespace_or_tab_as_delimiter](/docs/en/operations/settings/settings-formats.md/# input_format_csv_allow_whitespace_or_tab_as_delimiter) - Allow to use whitespace or tab as field delimiter in CSV strings. Default value - `false`.
+- [input_format_csv_allow_variable_number_of_columns](/docs/en/operations/settings/settings-formats.md/#input_format_csv_allow_variable_number_of_columns) - ignore extra columns in CSV input (if file has more columns than expected) and treat missing fields in CSV input as default values. Default value - `false`.
+- [input_format_csv_use_default_on_bad_values](/docs/en/operations/settings/settings-formats.md/#input_format_csv_use_default_on_bad_values) - Allow to set default value to column when CSV field deserialization failed on bad value. Default value - `false`.
## CSVWithNames {#csvwithnames}
@@ -1298,8 +1301,8 @@ For output it uses the following correspondence between ClickHouse types and BSO
| [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `\x04` array |
| [Named Tuple](/docs/en/sql-reference/data-types/tuple.md) | `\x03` document |
| [Map](/docs/en/sql-reference/data-types/map.md) | `\x03` document |
-| [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `\x10` int32 |
-| [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `\x05` binary, `\x00` binary subtype |
+| [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `\x10` int32 |
+| [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `\x05` binary, `\x00` binary subtype |
For input it uses the following correspondence between BSON types and ClickHouse types:
@@ -1309,7 +1312,7 @@ For input it uses the following correspondence between BSON types and ClickHouse
| `\x02` string | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) |
| `\x03` document | [Map](/docs/en/sql-reference/data-types/map.md)/[Named Tuple](/docs/en/sql-reference/data-types/tuple.md) |
| `\x04` array | [Array](/docs/en/sql-reference/data-types/array.md)/[Tuple](/docs/en/sql-reference/data-types/tuple.md) |
-| `\x05` binary, `\x00` binary subtype | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md)/[IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) |
+| `\x05` binary, `\x00` binary subtype | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md)/[IPv6](/docs/en/sql-reference/data-types/ipv6.md) |
| `\x05` binary, `\x02` old binary subtype | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) |
| `\x05` binary, `\x03` old uuid subtype | [UUID](/docs/en/sql-reference/data-types/uuid.md) |
| `\x05` binary, `\x04` uuid subtype | [UUID](/docs/en/sql-reference/data-types/uuid.md) |
@@ -1319,7 +1322,7 @@ For input it uses the following correspondence between BSON types and ClickHouse
| `\x0A` null value | [NULL](/docs/en/sql-reference/data-types/nullable.md) |
| `\x0D` JavaScript code | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) |
| `\x0E` symbol | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) |
-| `\x10` int32 | [Int32/UInt32](/docs/en/sql-reference/data-types/int-uint.md)/[Decimal32](/docs/en/sql-reference/data-types/decimal.md)/[IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md)/[Enum8/Enum16](/docs/en/sql-reference/data-types/enum.md) |
+| `\x10` int32 | [Int32/UInt32](/docs/en/sql-reference/data-types/int-uint.md)/[Decimal32](/docs/en/sql-reference/data-types/decimal.md)/[IPv4](/docs/en/sql-reference/data-types/ipv4.md)/[Enum8/Enum16](/docs/en/sql-reference/data-types/enum.md) |
| `\x12` int64 | [Int64/UInt64](/docs/en/sql-reference/data-types/int-uint.md)/[Decimal64](/docs/en/sql-reference/data-types/decimal.md)/[DateTime64](/docs/en/sql-reference/data-types/datetime64.md) |
Other BSON types are not supported. Also, it performs conversion between different integer types (for example, you can insert BSON int32 value into ClickHouse UInt8).
@@ -1514,6 +1517,23 @@ If setting [input_format_with_types_use_header](/docs/en/operations/settings/set
the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
:::
+## RowBinaryWithDefaults {#rowbinarywithdefaults}
+
+Similar to [RowBinary](#rowbinary), but with an extra byte before each column that indicates if default value should be used.
+
+Examples:
+
+```sql
+:) select * from format('RowBinaryWithDefaults', 'x UInt32 default 42, y UInt32', x'010001000000')
+
+┌──x─┬─y─┐
+│ 42 │ 1 │
+└────┴───┘
+```
+
+For column `x` there is only one byte `01` that indicates that default value should be used and no other data after this byte is provided.
+For column `y` data starts with byte `00` that indicates that column has actual value that should be read from the subsequent data `01000000`.
+
## RowBinary format settings {#row-binary-format-settings}
- [format_binary_max_string_size](/docs/en/operations/settings/settings-formats.md/#format_binary_max_string_size) - The maximum allowed size for String in RowBinary format. Default value - `1GiB`.
@@ -1669,8 +1689,8 @@ The table below shows supported data types and how they match ClickHouse [data t
| `ENUM` | [Enum(8/16)](/docs/en/sql-reference/data-types/enum.md) | `ENUM` |
| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` |
| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` |
-| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` |
-| `DATA` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `DATA` |
+| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `UINT32` |
+| `DATA` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `DATA` |
| `DATA` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `DATA` |
| `DATA` | [Decimal128/Decimal256](/docs/en/sql-reference/data-types/decimal.md) | `DATA` |
| `STRUCT(entries LIST(STRUCT(key Key, value Value)))` | [Map](/docs/en/sql-reference/data-types/map.md) | `STRUCT(entries LIST(STRUCT(key Key, value Value)))` |
@@ -1872,8 +1892,8 @@ The table below shows supported data types and how they match ClickHouse [data t
| `long (timestamp-millis)` \** | [DateTime64(3)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-millis)` \** |
| `long (timestamp-micros)` \** | [DateTime64(6)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-micros)` \** |
| `bytes (decimal)` \** | [DateTime64(N)](/docs/en/sql-reference/data-types/datetime.md) | `bytes (decimal)` \** |
-| `int` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `int` |
-| `fixed(16)` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `fixed(16)` |
+| `int` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `int` |
+| `fixed(16)` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `fixed(16)` |
| `bytes (decimal)` \** | [Decimal(P, S)](/docs/en/sql-reference/data-types/decimal.md) | `bytes (decimal)` \** |
| `string (uuid)` \** | [UUID](/docs/en/sql-reference/data-types/uuid.md) | `string (uuid)` \** |
| `fixed(16)` | [Int128/UInt128](/docs/en/sql-reference/data-types/int-uint.md) | `fixed(16)` |
@@ -2026,9 +2046,9 @@ The table below shows supported data types and how they match ClickHouse [data t
| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` |
| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` |
| `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` |
-| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` |
-| `FIXED_LENGTH_BYTE_ARRAY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `FIXED_LENGTH_BYTE_ARRAY` |
-| `FIXED_LENGTH_BYTE_ARRAY`, `BINARY` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `FIXED_LENGTH_BYTE_ARRAY` |
+| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `UINT32` |
+| `FIXED_LENGTH_BYTE_ARRAY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `FIXED_LENGTH_BYTE_ARRAY` |
+| `FIXED_LENGTH_BYTE_ARRAY`, `BINARY` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `FIXED_LENGTH_BYTE_ARRAY` |
Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested.
@@ -2082,7 +2102,7 @@ Special format for reading Parquet file metadata (https://parquet.apache.org/doc
- logical_type - column logical type
- compression - compression used for this column
- total_uncompressed_size - total uncompressed bytes size of the column, calculated as the sum of total_uncompressed_size of the column from all row groups
- - total_compressed_size - total compressed bytes size of the column, calculated as the sum of total_compressed_size of the column from all row groups
+ - total_compressed_size - total compressed bytes size of the column, calculated as the sum of total_compressed_size of the column from all row groups
- space_saved - percent of space saved by compression, calculated as (1 - total_compressed_size/total_uncompressed_size).
- encodings - the list of encodings used for this column
- row_groups - the list of row groups metadata with the next structure:
@@ -2229,9 +2249,9 @@ The table below shows supported data types and how they match ClickHouse [data t
| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` |
| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` |
| `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` |
-| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` |
-| `FIXED_SIZE_BINARY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `FIXED_SIZE_BINARY` |
-| `FIXED_SIZE_BINARY`, `BINARY` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `FIXED_SIZE_BINARY` |
+| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `UINT32` |
+| `FIXED_SIZE_BINARY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `FIXED_SIZE_BINARY` |
+| `FIXED_SIZE_BINARY`, `BINARY` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `FIXED_SIZE_BINARY` |
Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested.
@@ -2297,7 +2317,7 @@ The table below shows supported data types and how they match ClickHouse [data t
| `Struct` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `Struct` |
| `Map` | [Map](/docs/en/sql-reference/data-types/map.md) | `Map` |
| `Int` | [IPv4](/docs/en/sql-reference/data-types/int-uint.md) | `Int` |
-| `Binary` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `Binary` |
+| `Binary` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `Binary` |
| `Binary` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `Binary` |
| `Binary` | [Decimal256](/docs/en/sql-reference/data-types/decimal.md) | `Binary` |
@@ -2454,18 +2474,22 @@ In this format, all input data is read to a single value. It is possible to pars
The result is output in binary format without delimiters and escaping. If more than one value is output, the format is ambiguous, and it will be impossible to read the data back.
Below is a comparison of the formats `RawBLOB` and [TabSeparatedRaw](#tabseparatedraw).
+
`RawBLOB`:
- data is output in binary format, no escaping;
- there are no delimiters between values;
- no newline at the end of each value.
-[TabSeparatedRaw] (#tabseparatedraw):
+
+`TabSeparatedRaw`:
- data is output without escaping;
- the rows contain values separated by tabs;
- there is a line feed after the last value in every row.
The following is a comparison of the `RawBLOB` and [RowBinary](#rowbinary) formats.
+
`RawBLOB`:
- String fields are output without being prefixed by length.
+
`RowBinary`:
- String fields are represented as length in varint format (unsigned [LEB128] (https://en.wikipedia.org/wiki/LEB128)), followed by the bytes of the string.
@@ -2510,7 +2534,7 @@ ClickHouse supports reading and writing [MessagePack](https://msgpack.org/) data
| `uint 64` | [DateTime64](/docs/en/sql-reference/data-types/datetime.md) | `uint 64` |
| `fixarray`, `array 16`, `array 32` | [Array](/docs/en/sql-reference/data-types/array.md)/[Tuple](/docs/en/sql-reference/data-types/tuple.md) | `fixarray`, `array 16`, `array 32` |
| `fixmap`, `map 16`, `map 32` | [Map](/docs/en/sql-reference/data-types/map.md) | `fixmap`, `map 16`, `map 32` |
-| `uint 32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `uint 32` |
+| `uint 32` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `uint 32` |
| `bin 8` | [String](/docs/en/sql-reference/data-types/string.md) | `bin 8` |
| `int 8` | [Enum8](/docs/en/sql-reference/data-types/enum.md) | `int 8` |
| `bin 8` | [(U)Int128/(U)Int256](/docs/en/sql-reference/data-types/int-uint.md) | `bin 8` |
diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md
index 3a7f6d4d854..37821f0fee1 100644
--- a/docs/en/interfaces/http.md
+++ b/docs/en/interfaces/http.md
@@ -56,7 +56,7 @@ Connection: Close
Content-Type: text/tab-separated-values; charset=UTF-8
X-ClickHouse-Server-Display-Name: clickhouse.ru-central1.internal
X-ClickHouse-Query-Id: 5abe861c-239c-467f-b955-8a201abb8b7f
-X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
1
```
@@ -286,9 +286,9 @@ Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you
You can receive information about the progress of a query in `X-ClickHouse-Progress` response headers. To do this, enable [send_progress_in_http_headers](../operations/settings/settings.md#settings-send_progress_in_http_headers). Example of the header sequence:
``` text
-X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128"}
-X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128"}
-X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128"}
+X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128","peak_memory_usage":"4371480"}
+X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128","peak_memory_usage":"13621616"}
+X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128","peak_memory_usage":"23155600"}
```
Possible header fields:
@@ -416,7 +416,7 @@ $ curl -v 'http://localhost:8123/predefined_query'
< X-ClickHouse-Format: Template
< X-ClickHouse-Timezone: Asia/Shanghai
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
# HELP "Query" "Number of executing queries"
# TYPE "Query" counter
@@ -581,7 +581,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
* Connection #0 to host localhost left intact
Say Hi!%
@@ -621,7 +621,7 @@ $ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler'
< Content-Type: text/plain; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
* Connection #0 to host localhost left intact
%
@@ -673,7 +673,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
Absolute Path File
* Connection #0 to host localhost left intact
@@ -692,7 +692,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
Relative Path File
* Connection #0 to host localhost left intact
diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md
index c3ddee07d0b..62f931a76b4 100644
--- a/docs/en/operations/backup.md
+++ b/docs/en/operations/backup.md
@@ -30,7 +30,7 @@ description: In order to effectively mitigate possible human errors, you should
```
:::note ALL
-`ALL` is only applicable to the `RESTORE` command prior to version 23.4 of Clickhouse.
+Prior to version 23.4 of ClickHouse, `ALL` was only applicable to the `RESTORE` command.
:::
## Background
diff --git a/docs/en/operations/configuration-files.md b/docs/en/operations/configuration-files.md
index b3583e156ad..d1d9fa542ab 100644
--- a/docs/en/operations/configuration-files.md
+++ b/docs/en/operations/configuration-files.md
@@ -6,32 +6,43 @@ sidebar_label: Configuration Files
# Configuration Files
-ClickHouse supports multi-file configuration management. The main server configuration file is `/etc/clickhouse-server/config.xml` or `/etc/clickhouse-server/config.yaml`. Other files must be in the `/etc/clickhouse-server/config.d` directory. Note, that any configuration file can be written either in XML or YAML, but mixing formats in one file is not supported. For example, you can have main configs as `config.xml` and `users.xml` and write additional files in `config.d` and `users.d` directories in `.yaml`.
+The ClickHouse server can be configured with configuration files in XML or YAML syntax. In most installation types, the ClickHouse server runs with `/etc/clickhouse-server/config.xml` as default configuration file but it is also possible to specify the location of the configuration file manually at server startup using command line option `--config-file=` or `-C`. Additional configuration files may be placed into directory `config.d/` relative to the main configuration file, for example into directory `/etc/clickhouse-server/config.d/`. Files in this directory and the main configuration are merged in a preprocessing step before the configuration is applied in ClickHouse server. Configuration files are merged in alphabetical order. To simplify updates and improve modularization, it is best practice to keep the default `config.xml` file unmodified and place additional customization into `config.d/`.
-All XML files should have the same root element, usually ``. As for YAML, `clickhouse:` should not be present, the parser will insert it automatically.
+It is possible to mix XML and YAML configuration files, for example you could have a main configuration file `config.xml` and additional configuration files `config.d/network.xml`, `config.d/timezone.yaml` and `config.d/keeper.yaml`. Mixing XML and YAML within a single configuration file is not supported. XML configuration files should use `...` as top-level tag. In YAML configuration files, `clickhouse:` is optional, the parser inserts it implicitly if absent.
-## Override {#override}
+## Overriding Configuration {#override}
-Some settings specified in the main configuration file can be overridden in other configuration files:
+The merge of configuration files behaves as one intuitively expects: The contents of both files are combined recursively, children with the same name are replaced by the element of the more specific configuration file. The merge can be customized using attributes `replace` and `remove`.
+- Attribute `replace` means that the element is replaced by the specified one.
+- Attribute `remove` means that the element is deleted.
-- The `replace` or `remove` attributes can be specified for the elements of these configuration files.
-- If neither is specified, it combines the contents of elements recursively, replacing values of duplicate children.
-- If `replace` is specified, it replaces the entire element with the specified one.
-- If `remove` is specified, it deletes the element.
+To specify that a value of an element should be replaced by the value of an environment variable, you can use attribute `from_env`.
-You can also declare attributes as coming from environment variables by using `from_env="VARIABLE_NAME"`:
+Example with `$MAX_QUERY_SIZE = 150000`:
```xml
-
-
-
-
-
+
+
+
+
+
```
-## Substitution {#substitution}
+which is equal to
+
+``` xml
+
+
+
+ 150000
+
+
+
+```
+
+## Substituting Configuration {#substitution}
The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/clickhouse/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](../operations/server-configuration-parameters/settings.md#macros)).
@@ -54,6 +65,40 @@ XML substitution example:
Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at `/path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element.
+## Encrypting Configuration {#encryption}
+
+You can use symmetric encryption to encrypt a configuration element, for example, a password field. To do so, first configure the [encryption codec](../sql-reference/statements/create/table.md#encryption-codecs), then add attribute `encryption_codec` with the name of the encryption codec as value to the element to encrypt.
+
+Unlike attributes `from_zk`, `from_env` and `incl` (or element `include`), no substitution, i.e. decryption of the encrypted value, is performed in the preprocessed file. Decryption happens only at runtime in the server process.
+
+Example:
+
+```xml
+
+
+
+ 00112233445566778899aabbccddeeff
+
+
+
+ admin
+ 961F000000040000000000EEDDEF4F453CFE6457C4234BD7C09258BD651D85
+
+
+```
+
+To get the encrypted value `encrypt_decrypt` example application may be used.
+
+Example:
+
+``` bash
+./encrypt_decrypt /etc/clickhouse-server/config.xml -e AES_128_GCM_SIV abcd
+```
+
+``` text
+961F000000040000000000EEDDEF4F453CFE6457C4234BD7C09258BD651D85
+```
+
## User Settings {#user-settings}
The `config.xml` file can specify a separate config with user settings, profiles, and quotas. The relative path to this config is set in the `users_config` element. By default, it is `users.xml`. If `users_config` is omitted, the user settings, profiles, and quotas are specified directly in `config.xml`.
diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md
index a521a369721..02f52b6f8bf 100644
--- a/docs/en/operations/named-collections.md
+++ b/docs/en/operations/named-collections.md
@@ -50,7 +50,7 @@ To manage named collections with DDL a user must have the `named_control_collect
```
:::tip
-In the above example the `passowrd_sha256_hex` value is the hexadecimal representation of the SHA256 hash of the password. This configuration for the user `default` has the attribute `replace=true` as in the default configuration has a plain text `password` set, and it is not possible to have both plain text and sha256 hex passwords set for a user.
+In the above example the `password_sha256_hex` value is the hexadecimal representation of the SHA256 hash of the password. This configuration for the user `default` has the attribute `replace=true` as in the default configuration has a plain text `password` set, and it is not possible to have both plain text and sha256 hex passwords set for a user.
:::
## Storing named collections in configuration files
diff --git a/docs/en/operations/optimizing-performance/index.md b/docs/en/operations/optimizing-performance/index.md
deleted file mode 100644
index 83e9430ed27..00000000000
--- a/docs/en/operations/optimizing-performance/index.md
+++ /dev/null
@@ -1,9 +0,0 @@
----
-slug: /en/operations/optimizing-performance/
-sidebar_label: Optimizing Performance
-sidebar_position: 52
----
-
-# Optimizing Performance
-
-- [Sampling query profiler](../../operations/optimizing-performance/sampling-query-profiler.md)
diff --git a/docs/en/operations/server-configuration-parameters/index.md b/docs/en/operations/server-configuration-parameters/index.md
deleted file mode 100644
index d4b941c0819..00000000000
--- a/docs/en/operations/server-configuration-parameters/index.md
+++ /dev/null
@@ -1,16 +0,0 @@
----
-slug: /en/operations/server-configuration-parameters/
-sidebar_position: 54
-sidebar_label: Server Configuration Parameters
-pagination_next: en/operations/server-configuration-parameters/settings
----
-
-# Server Configuration Parameters
-
-This section contains descriptions of server settings that cannot be changed at the session or query level.
-
-These settings are stored in the `config.xml` file on the ClickHouse server.
-
-Other settings are described in the “[Settings](../../operations/settings/index.md#session-settings-intro)” section.
-
-Before studying the settings, read the [Configuration files](../../operations/configuration-files.md#configuration_files) section and note the use of substitutions (the `incl` and `optional` attributes).
diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md
index 83756097cfa..a6ae517e401 100644
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@@ -1,11 +1,19 @@
---
slug: /en/operations/server-configuration-parameters/settings
sidebar_position: 57
-sidebar_label: Server Settings
+sidebar_label: Global Server Settings
description: This section contains descriptions of server settings that cannot be changed at the session or query level.
---
-# Server Settings
+# Global Server Settings
+
+This section contains descriptions of server settings that cannot be changed at the session or query level.
+
+These settings are stored in the `config.xml` file on the ClickHouse server.
+
+Other settings are described in the “[Settings](../../operations/settings/index.md#session-settings-intro)” section.
+
+Before studying the settings, read the [Configuration files](../../operations/configuration-files.md#configuration_files) section and note the use of substitutions (the `incl` and `optional` attributes).
## allow_use_jemalloc_memory
@@ -1193,13 +1201,58 @@ Keys:
- `console` – Send `log` and `errorlog` to the console instead of file. To enable, set to `1` or `true`.
- `stream_compress` – Compress `log` and `errorlog` with `lz4` stream compression. To enable, set to `1` or `true`.
+Both log and error log file names (only file names, not directories) support date and time format specifiers.
+
+**Format specifiers**
+Using the following format specifiers, you can define a pattern for the resulting file name. “Example” column shows possible results for `2023-07-06 18:32:07`.
+
+| Specifier | Description | Example |
+|-------------|---------------------------------------------------------------------------------------------------------------------|--------------------------|
+| %% | Literal % | % |
+| %n | New-line character | |
+| %t | Horizontal tab character | |
+| %Y | Year as a decimal number, e.g. 2017 | 2023 |
+| %y | Last 2 digits of year as a decimal number (range [00,99]) | 23 |
+| %C | First 2 digits of year as a decimal number (range [00,99]) | 20 |
+| %G | Four-digit [ISO 8601 week-based year](https://en.wikipedia.org/wiki/ISO_8601#Week_dates), i.e. the year that contains the specified week. Normally useful only with %V | 2023 |
+| %g | Last 2 digits of [ISO 8601 week-based year](https://en.wikipedia.org/wiki/ISO_8601#Week_dates), i.e. the year that contains the specified week. | 23 |
+| %b | Abbreviated month name, e.g. Oct (locale dependent) | Jul |
+| %h | Synonym of %b | Jul |
+| %B | Full month name, e.g. October (locale dependent) | July |
+| %m | Month as a decimal number (range [01,12]) | 07 |
+| %U | Week of the year as a decimal number (Sunday is the first day of the week) (range [00,53]) | 27 |
+| %W | Week of the year as a decimal number (Monday is the first day of the week) (range [00,53]) | 27 |
+| %V | ISO 8601 week number (range [01,53]) | 27 |
+| %j | Day of the year as a decimal number (range [001,366]) | 187 |
+| %d | Day of the month as a zero-padded decimal number (range [01,31]). Single digit is preceded by zero. | 06 |
+| %e | Day of the month as a space-padded decimal number (range [1,31]). Single digit is preceded by a space. | 6 |
+| %a | Abbreviated weekday name, e.g. Fri (locale dependent) | Thu |
+| %A | Full weekday name, e.g. Friday (locale dependent) | Thursday |
+| %w | Weekday as a integer number with Sunday as 0 (range [0-6]) | 4 |
+| %u | Weekday as a decimal number, where Monday is 1 (ISO 8601 format) (range [1-7]) | 4 |
+| %H | Hour as a decimal number, 24 hour clock (range [00-23]) | 18 |
+| %I | Hour as a decimal number, 12 hour clock (range [01,12]) | 06 |
+| %M | Minute as a decimal number (range [00,59]) | 32 |
+| %S | Second as a decimal number (range [00,60]) | 07 |
+| %c | Standard date and time string, e.g. Sun Oct 17 04:41:13 2010 (locale dependent) | Thu Jul 6 18:32:07 2023 |
+| %x | Localized date representation (locale dependent) | 07/06/23 |
+| %X | Localized time representation, e.g. 18:40:20 or 6:40:20 PM (locale dependent) | 18:32:07 |
+| %D | Short MM/DD/YY date, equivalent to %m/%d/%y | 07/06/23 |
+| %F | Short YYYY-MM-DD date, equivalent to %Y-%m-%d | 2023-07-06 |
+| %r | Localized 12-hour clock time (locale dependent) | 06:32:07 PM |
+| %R | Equivalent to "%H:%M" | 18:32 |
+| %T | Equivalent to "%H:%M:%S" (the ISO 8601 time format) | 18:32:07 |
+| %p | Localized a.m. or p.m. designation (locale dependent) | PM |
+| %z | Offset from UTC in the ISO 8601 format (e.g. -0430), or no characters if the time zone information is not available | +0800 |
+| %Z | Locale-dependent time zone name or abbreviation, or no characters if the time zone information is not available | Z AWST |
+
**Example**
``` xml
trace
- /var/log/clickhouse-server/clickhouse-server.log
- /var/log/clickhouse-server/clickhouse-server.err.log
+ /var/log/clickhouse-server/clickhouse-server-%F-%T.log
+ /var/log/clickhouse-server/clickhouse-server-%F-%T.err.log
1000M
10
true
@@ -1594,7 +1647,7 @@ Keys for server/client settings:
- requireTLSv1_2 (default: false) – Require a TLSv1.2 connection. Acceptable values: `true`, `false`.
- fips (default: false) – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS.
- privateKeyPassphraseHandler (default: `KeyConsoleHandler`)– Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ``, `KeyFileHandler`, `test`, ``.
-- invalidCertificateHandler (default: `ConsoleCertificateHandler`) – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` ConsoleCertificateHandler ` .
+- invalidCertificateHandler (default: `RejectCertificateHandler`) – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` RejectCertificateHandler ` .
- disableProtocols (default: "") – Protocols that are not allowed to use.
- preferServerCiphers (default: false) – Preferred server ciphers on the client.
@@ -1967,6 +2020,10 @@ The time zone is necessary for conversions between String and DateTime formats w
Asia/Istanbul
```
+**See also**
+
+- [session_timezone](../settings/settings.md#session_timezone)
+
## tcp_port {#server_configuration_parameters-tcp_port}
Port for communicating with clients over the TCP protocol.
@@ -2108,7 +2165,13 @@ This section contains the following parameters:
- `operation_timeout_ms` — Maximum timeout for one operation in milliseconds.
- `root` — The [znode](http://zookeeper.apache.org/doc/r3.5.5/zookeeperOver.html#Nodes+and+ephemeral+nodes) that is used as the root for znodes used by the ClickHouse server. Optional.
- `identity` — User and password, that can be required by ZooKeeper to give access to requested znodes. Optional.
-
+- zookeeper_load_balancing - Specifies the algorithm of ZooKeeper node selection.
+ * random - randomly selects one of ZooKeeper nodes.
+ * in_order - selects the first ZooKeeper node, if it's not available then the second, and so on.
+ * nearest_hostname - selects a ZooKeeper node with a hostname that is most similar to the server’s hostname.
+ * first_or_random - selects the first ZooKeeper node, if it's not available then randomly selects one of remaining ZooKeeper nodes.
+ * round_robin - selects the first ZooKeeper node, if reconnection happens selects the next.
+
**Example configuration**
``` xml
@@ -2127,6 +2190,8 @@ This section contains the following parameters:
/path/to/zookeeper/node
user:password
+
+ random
```
diff --git a/docs/en/operations/settings/index.md b/docs/en/operations/settings/index.md
index eb1d5db5676..6863d7f3191 100644
--- a/docs/en/operations/settings/index.md
+++ b/docs/en/operations/settings/index.md
@@ -7,90 +7,16 @@ pagination_next: en/operations/settings/settings
# Settings Overview
-There are multiple ways to define ClickHouse settings. Settings are configured in layers, and each subsequent layer redefines the previous values of a setting.
+There are two main groups of ClickHouse settings:
-The order of priority for defining a setting is:
+- Global server settings
+- Query-level settings
-1. Settings in the `users.xml` server configuration file
+The main distinction between global server settings and query-level settings is that
+global server settings must be set in configuration files while query-level settings
+can be set in configuration files or with SQL queries.
- - Set in the element ``.
+Read about [global server settings](/docs/en/operations/server-configuration-parameters/settings.md) to learn more about configuring your ClickHouse server at the global server level.
-2. Session settings
+Read about [query-level settings](/docs/en/operations/settings/settings-query-level.md) to learn more about configuring your ClickHouse server at the query-level.
- - Send `SET setting=value` from the ClickHouse console client in interactive mode.
- Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` HTTP parameter.
-
-3. Query settings
-
- - When starting the ClickHouse console client in non-interactive mode, set the startup parameter `--setting=value`.
- - When using the HTTP API, pass CGI parameters (`URL?setting_1=value&setting_2=value...`).
- - Define settings in the [SETTINGS](../../sql-reference/statements/select/index.md#settings-in-select-query) clause of the SELECT query. The setting value is applied only to that query and is reset to the default or previous value after the query is executed.
-
-View the [Settings](./settings.md) page for a description of the ClickHouse settings.
-
-## Converting a Setting to its Default Value
-
-If you change a setting and would like to revert it back to its default value, set the value to `DEFAULT`. The syntax looks like:
-
-```sql
-SET setting_name = DEFAULT
-```
-
-For example, the default value of `max_insert_block_size` is 1048449. Suppose you change its value to 100000:
-
-```sql
-SET max_insert_block_size=100000;
-
-SELECT value FROM system.settings where name='max_insert_block_size';
-```
-
-The response is:
-
-```response
-┌─value──┐
-│ 100000 │
-└────────┘
-```
-
-The following command sets its value back to 1048449:
-
-```sql
-SET max_insert_block_size=DEFAULT;
-
-SELECT value FROM system.settings where name='max_insert_block_size';
-```
-
-The setting is now back to its default:
-
-```response
-┌─value───┐
-│ 1048449 │
-└─────────┘
-```
-
-
-## Custom Settings {#custom_settings}
-
-In addition to the common [settings](../../operations/settings/settings.md), users can define custom settings.
-
-A custom setting name must begin with one of predefined prefixes. The list of these prefixes must be declared in the [custom_settings_prefixes](../../operations/server-configuration-parameters/settings.md#custom_settings_prefixes) parameter in the server configuration file.
-
-```xml
-custom_
-```
-
-To define a custom setting use `SET` command:
-
-```sql
-SET custom_a = 123;
-```
-
-To get the current value of a custom setting use `getSetting()` function:
-
-```sql
-SELECT getSetting('custom_a');
-```
-
-**See Also**
-
-- [Server Configuration Settings](../../operations/server-configuration-parameters/settings.md)
diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md
index 1b22a6d1223..ee8e0d547b8 100644
--- a/docs/en/operations/settings/settings-formats.md
+++ b/docs/en/operations/settings/settings-formats.md
@@ -242,6 +242,26 @@ See also:
- [DateTime data type.](../../sql-reference/data-types/datetime.md)
- [Functions for working with dates and times.](../../sql-reference/functions/date-time-functions.md)
+## interval_output_format {#interval_output_format}
+
+Allows choosing different output formats of the text representation of interval types.
+
+Possible values:
+
+- `kusto` - KQL-style output format.
+
+ ClickHouse outputs intervals in [KQL format](https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-timespan-format-strings#the-constant-c-format-specifier). For example, `toIntervalDay(2)` would be formatted as `2.00:00:00`. Please note that for interval types of varying length (ie. `IntervalMonth` and `IntervalYear`) the average number of seconds per interval is taken into account.
+
+- `numeric` - Numeric output format.
+
+ ClickHouse outputs intervals as their underlying numeric representation. For example, `toIntervalDay(2)` would be formatted as `2`.
+
+Default value: `numeric`.
+
+See also:
+
+- [Interval](../../sql-reference/data-types/special-data-types/interval.md)
+
## input_format_ipv4_default_on_conversion_error {#input_format_ipv4_default_on_conversion_error}
Deserialization of IPv4 will use default values instead of throwing exception on conversion error.
@@ -931,6 +951,11 @@ Result
```text
" string "
```
+### input_format_csv_allow_variable_number_of_columns {#input_format_csv_allow_variable_number_of_columns}
+
+ignore extra columns in CSV input (if file has more columns than expected) and treat missing fields in CSV input as default values.
+
+Disabled by default.
### input_format_csv_allow_whitespace_or_tab_as_delimiter {#input_format_csv_allow_whitespace_or_tab_as_delimiter}
@@ -964,6 +989,28 @@ Result
a b
```
+### input_format_csv_use_default_on_bad_values {#input_format_csv_use_default_on_bad_values}
+
+Allow to set default value to column when CSV field deserialization failed on bad value
+
+Default value: `false`.
+
+**Examples**
+
+Query
+
+```bash
+./clickhouse local -q "create table test_tbl (x String, y UInt32, z Date) engine=MergeTree order by x"
+echo 'a,b,c' | ./clickhouse local -q "INSERT INTO test_tbl SETTINGS input_format_csv_use_default_on_bad_values=true FORMAT CSV"
+./clickhouse local -q "select * from test_tbl"
+```
+
+Result
+
+```text
+a 0 1971-01-01
+```
+
## Values format settings {#values-format-settings}
### input_format_values_interpret_expressions {#input_format_values_interpret_expressions}
@@ -1300,6 +1347,17 @@ Default value: 0.
Sets [Confluent Schema Registry](https://docs.confluent.io/current/schema-registry/index.html) URL to use with [AvroConfluent](../../interfaces/formats.md/#data-format-avro-confluent) format.
+Format:
+``` text
+http://[user:password@]machine[:port]"
+```
+
+Examples:
+``` text
+http://registry.example.com:8081
+http://admin:secret@registry.example.com:8081
+```
+
Default value: `Empty`.
### output_format_avro_codec {#output_format_avro_codec}
diff --git a/docs/en/operations/settings/settings-query-level.md b/docs/en/operations/settings/settings-query-level.md
new file mode 100644
index 00000000000..81cc2294a4c
--- /dev/null
+++ b/docs/en/operations/settings/settings-query-level.md
@@ -0,0 +1,217 @@
+---
+sidebar_label: Query-level Settings
+title: Query-level Settings
+slug: /en/operations/settings/query-level
+---
+
+There are multiple ways to set ClickHouse query-level settings. Settings are configured in layers, and each subsequent layer redefines the previous values of a setting.
+
+The order of priority for defining a setting is:
+
+1. Applying a setting to a user directly, or within a settings profile
+
+ - SQL (recommended)
+ - adding one or more XML or YAML files to `/etc/clickhouse-server/users.d`
+
+2. Session settings
+
+ - Send `SET setting=value` from the ClickHouse Cloud SQL console or
+ `clickhouse client` in interactive mode. Similarly, you can use ClickHouse
+ sessions in the HTTP protocol. To do this, you need to specify the
+ `session_id` HTTP parameter.
+
+3. Query settings
+
+ - When starting `clickhouse client` in non-interactive mode, set the startup
+ parameter `--setting=value`.
+ - When using the HTTP API, pass CGI parameters (`URL?setting_1=value&setting_2=value...`).
+ - Define settings in the
+ [SETTINGS](../../sql-reference/statements/select/index.md#settings-in-select-query)
+ clause of the SELECT query. The setting value is applied only to that query
+ and is reset to the default or previous value after the query is executed.
+
+## Examples
+
+These examples all set the value of the `async_insert` setting to `1`, and
+show how to examine the settings in a running system.
+
+### Using SQL to apply a setting to a user directly
+
+This creates the user `ingester` with the setting `async_inset = 1`:
+
+```sql
+CREATE USER ingester
+IDENTIFIED WITH sha256_hash BY '7e099f39b84ea79559b3e85ea046804e63725fd1f46b37f281276aae20f86dc3'
+# highlight-next-line
+SETTINGS async_insert = 1
+```
+
+#### Examine the settings profile and assignment
+
+```sql
+SHOW ACCESS
+```
+
+```response
+┌─ACCESS─────────────────────────────────────────────────────────────────────────────┐
+│ ... │
+# highlight-next-line
+│ CREATE USER ingester IDENTIFIED WITH sha256_password SETTINGS async_insert = true │
+│ ... │
+└────────────────────────────────────────────────────────────────────────────────────┘
+```
+### Using SQL to create a settings profile and assign to a user
+
+This creates the profile `log_ingest` with the setting `async_inset = 1`:
+
+```sql
+CREATE
+SETTINGS PROFILE log_ingest SETTINGS async_insert = 1
+```
+
+This creates the user `ingester` and assigns the user the settings profile `log_ingest`:
+
+```sql
+CREATE USER ingester
+IDENTIFIED WITH sha256_hash BY '7e099f39b84ea79559b3e85ea046804e63725fd1f46b37f281276aae20f86dc3'
+# highlight-next-line
+SETTINGS PROFILE log_ingest
+```
+
+
+### Using XML to create a settings profile and user
+
+```xml title=/etc/clickhouse-server/users.d/users.xml
+
+# highlight-start
+
+
+ 1
+
+
+# highlight-end
+
+
+
+ 7e099f39b84ea79559b3e85ea046804e63725fd1f46b37f281276aae20f86dc3
+# highlight-start
+ log_ingest
+# highlight-end
+
+
+ 7e099f39b84ea79559b3e85ea046804e63725fd1f46b37f281276aae20f86dc3
+ 1
+ 1
+
+
+
+```
+
+#### Examine the settings profile and assignment
+
+```sql
+SHOW ACCESS
+```
+
+```response
+┌─ACCESS─────────────────────────────────────────────────────────────────────────────┐
+│ CREATE USER default IDENTIFIED WITH sha256_password │
+# highlight-next-line
+│ CREATE USER ingester IDENTIFIED WITH sha256_password SETTINGS PROFILE log_ingest │
+│ CREATE SETTINGS PROFILE default │
+# highlight-next-line
+│ CREATE SETTINGS PROFILE log_ingest SETTINGS async_insert = true │
+│ CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1 │
+│ ... │
+└────────────────────────────────────────────────────────────────────────────────────┘
+```
+
+### Assign a setting to a session
+
+```sql
+SET async_insert =1;
+SELECT value FROM system.settings where name='async_insert';
+```
+
+```response
+┌─value──┐
+│ 1 │
+└────────┘
+```
+
+### Assign a setting during a query
+
+```sql
+INSERT INTO YourTable
+# highlight-next-line
+SETTINGS async_insert=1
+VALUES (...)
+```
+
+
+## Converting a Setting to its Default Value
+
+If you change a setting and would like to revert it back to its default value, set the value to `DEFAULT`. The syntax looks like:
+
+```sql
+SET setting_name = DEFAULT
+```
+
+For example, the default value of `async_insert` is `0`. Suppose you change its value to `1`:
+
+```sql
+SET async_insert = 1;
+
+SELECT value FROM system.settings where name='async_insert';
+```
+
+The response is:
+
+```response
+┌─value──┐
+│ 1 │
+└────────┘
+```
+
+The following command sets its value back to 0:
+
+```sql
+SET async_insert = DEFAULT;
+
+SELECT value FROM system.settings where name='async_insert';
+```
+
+The setting is now back to its default:
+
+```response
+┌─value───┐
+│ 0 │
+└─────────┘
+```
+
+## Custom Settings {#custom_settings}
+
+In addition to the common [settings](../../operations/settings/settings.md), users can define custom settings.
+
+A custom setting name must begin with one of predefined prefixes. The list of these prefixes must be declared in the [custom_settings_prefixes](../../operations/server-configuration-parameters/settings.md#custom_settings_prefixes) parameter in the server configuration file.
+
+```xml
+custom_
+```
+
+To define a custom setting use `SET` command:
+
+```sql
+SET custom_a = 123;
+```
+
+To get the current value of a custom setting use `getSetting()` function:
+
+```sql
+SELECT getSetting('custom_a');
+```
+
+**See Also**
+
+- View the [Settings](./settings.md) page for a description of the ClickHouse settings.
+- [Global server settings](../../operations/server-configuration-parameters/settings.md)
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index f0668c1d58f..8dfb6c0d225 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -17,7 +17,8 @@ Default value: 0.
**Example**
``` sql
-insert into table_1 values (1, 'a'), (2, 'bb'), (3, 'ccc'), (4, 'dddd');
+INSERT INTO table_1 VALUES (1, 'a'), (2, 'bb'), (3, 'ccc'), (4, 'dddd');
+SELECT * FROM table_1;
```
```response
┌─x─┬─y────┐
@@ -30,7 +31,7 @@ insert into table_1 values (1, 'a'), (2, 'bb'), (3, 'ccc'), (4, 'dddd');
```sql
SELECT *
FROM table_1
-SETTINGS additional_table_filters = (('table_1', 'x != 2'))
+SETTINGS additional_table_filters = {'table_1': 'x != 2'}
```
```response
┌─x─┬─y────┐
@@ -50,7 +51,8 @@ Default value: `''`.
**Example**
``` sql
-insert into table_1 values (1, 'a'), (2, 'bb'), (3, 'ccc'), (4, 'dddd');
+INSERT INTO table_1 VALUES (1, 'a'), (2, 'bb'), (3, 'ccc'), (4, 'dddd');
+SElECT * FROM table_1;
```
```response
┌─x─┬─y────┐
@@ -535,6 +537,8 @@ Possible values:
The first phase of a grace join reads the right table and splits it into N buckets depending on the hash value of key columns (initially, N is `grace_hash_join_initial_buckets`). This is done in a way to ensure that each bucket can be processed independently. Rows from the first bucket are added to an in-memory hash table while the others are saved to disk. If the hash table grows beyond the memory limit (e.g., as set by [`max_bytes_in_join`](/docs/en/operations/settings/query-complexity.md/#settings-max_bytes_in_join)), the number of buckets is increased and the assigned bucket for each row. Any rows which don’t belong to the current bucket are flushed and reassigned.
+ Supports `INNER/LEFT/RIGHT/FULL ALL/ANY JOIN`.
+
- hash
[Hash join algorithm](https://en.wikipedia.org/wiki/Hash_join) is used. The most generic implementation that supports all combinations of kind and strictness and multiple join keys that are combined with `OR` in the `JOIN ON` section.
@@ -1322,7 +1326,7 @@ Connection pool size for PostgreSQL table engine and database engine.
Default value: 16
-## postgresql_connection_pool_size {#postgresql-connection-pool-size}
+## postgresql_connection_pool_wait_timeout {#postgresql-connection-pool-wait-timeout}
Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.
@@ -2941,7 +2945,7 @@ Default value: `0`.
## mutations_sync {#mutations_sync}
-Allows to execute `ALTER TABLE ... UPDATE|DELETE` queries ([mutations](../../sql-reference/statements/alter/index.md#mutations)) synchronously.
+Allows to execute `ALTER TABLE ... UPDATE|DELETE|MATERIALIZE INDEX|MATERIALIZE PROJECTION|MATERIALIZE COLUMN` queries ([mutations](../../sql-reference/statements/alter/index.md#mutations)) synchronously.
Possible values:
@@ -3201,6 +3205,40 @@ ENGINE = Log
└──────────────────────────────────────────────────────────────────────────┘
```
+## default_temporary_table_engine {#default_temporary_table_engine}
+
+Same as [default_table_engine](#default_table_engine) but for temporary tables.
+
+Default value: `Memory`.
+
+In this example, any new temporary table that does not specify an `Engine` will use the `Log` table engine:
+
+Query:
+
+```sql
+SET default_temporary_table_engine = 'Log';
+
+CREATE TEMPORARY TABLE my_table (
+ x UInt32,
+ y UInt32
+);
+
+SHOW CREATE TEMPORARY TABLE my_table;
+```
+
+Result:
+
+```response
+┌─statement────────────────────────────────────────────────────────────────┐
+│ CREATE TEMPORARY TABLE default.my_table
+(
+ `x` UInt32,
+ `y` UInt32
+)
+ENGINE = Log
+└──────────────────────────────────────────────────────────────────────────┘
+```
+
## data_type_default_nullable {#data_type_default_nullable}
Allows data types without explicit modifiers [NULL or NOT NULL](../../sql-reference/statements/create/table.md/#null-modifiers) in column definition will be [Nullable](../../sql-reference/data-types/nullable.md/#data_type-nullable).
@@ -3430,6 +3468,12 @@ Possible values:
Default value: `0`.
+## disable_url_encoding {#disable_url_encoding}
+
+Allows to disable decoding/encoding path in uri in [URL](../../engines/table-engines/special/url.md) engine tables.
+
+Disabled by default.
+
## database_atomic_wait_for_drop_and_detach_synchronously {#database_atomic_wait_for_drop_and_detach_synchronously}
Adds a modifier `SYNC` to all `DROP` and `DETACH` queries.
@@ -3501,7 +3545,7 @@ Possible values:
- Any positive integer.
- 0 - Disabled (infinite timeout).
-Default value: 180.
+Default value: 30.
## http_receive_timeout {#http_receive_timeout}
@@ -3512,7 +3556,7 @@ Possible values:
- Any positive integer.
- 0 - Disabled (infinite timeout).
-Default value: 180.
+Default value: 30.
## check_query_single_value_result {#check_query_single_value_result}
@@ -4251,6 +4295,69 @@ Default value: `0`.
Use this setting only for backward compatibility if your use cases depend on old syntax.
:::
+## session_timezone {#session_timezone}
+
+Sets the implicit time zone of the current session or query.
+The implicit time zone is the time zone applied to values of type DateTime/DateTime64 which have no explicitly specified time zone.
+The setting takes precedence over the globally configured (server-level) implicit time zone.
+A value of '' (empty string) means that the implicit time zone of the current session or query is equal to the [server time zone](../server-configuration-parameters/settings.md#server_configuration_parameters-timezone).
+
+You can use functions `timeZone()` and `serverTimeZone()` to get the session time zone and server time zone.
+
+Possible values:
+
+- Any time zone name from `system.time_zones`, e.g. `Europe/Berlin`, `UTC` or `Zulu`
+
+Default value: `''`.
+
+Examples:
+
+```sql
+SELECT timeZone(), serverTimeZone() FORMAT TSV
+
+Europe/Berlin Europe/Berlin
+```
+
+```sql
+SELECT timeZone(), serverTimeZone() SETTINGS session_timezone = 'Asia/Novosibirsk' FORMAT TSV
+
+Asia/Novosibirsk Europe/Berlin
+```
+
+Assign session time zone 'America/Denver' to the inner DateTime without explicitly specified time zone:
+
+```sql
+SELECT toDateTime64(toDateTime64('1999-12-12 23:23:23.123', 3), 3, 'Europe/Zurich') SETTINGS session_timezone = 'America/Denver' FORMAT TSV
+
+1999-12-13 07:23:23.123
+```
+
+:::warning
+Not all functions that parse DateTime/DateTime64 respect `session_timezone`. This can lead to subtle errors.
+See the following example and explanation.
+:::
+
+```sql
+CREATE TABLE test_tz (`d` DateTime('UTC')) ENGINE = Memory AS SELECT toDateTime('2000-01-01 00:00:00', 'UTC');
+
+SELECT *, timeZone() FROM test_tz WHERE d = toDateTime('2000-01-01 00:00:00') SETTINGS session_timezone = 'Asia/Novosibirsk'
+0 rows in set.
+
+SELECT *, timeZone() FROM test_tz WHERE d = '2000-01-01 00:00:00' SETTINGS session_timezone = 'Asia/Novosibirsk'
+┌───────────────────d─┬─timeZone()───────┐
+│ 2000-01-01 00:00:00 │ Asia/Novosibirsk │
+└─────────────────────┴──────────────────┘
+```
+
+This happens due to different parsing pipelines:
+
+- `toDateTime()` without explicitly given time zone used in the first `SELECT` query honors setting `session_timezone` and the global time zone.
+- In the second query, a DateTime is parsed from a String, and inherits the type and time zone of the existing column`d`. Thus, setting `session_timezone` and the global time zone are not honored.
+
+**See also**
+
+- [timezone](../server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
+
## final {#final}
Automatically applies [FINAL](../../sql-reference/statements/select/from.md#final-modifier) modifier to all tables in a query, to tables where [FINAL](../../sql-reference/statements/select/from.md#final-modifier) is applicable, including joined tables and tables in sub-queries, and
@@ -4425,6 +4532,7 @@ This setting allows to specify renaming pattern for files processed by `file` ta
### Placeholders
+- `%a` — Full original filename (e.g., "sample.csv").
- `%f` — Original filename without extension (e.g., "sample").
- `%e` — Original file extension with dot (e.g., ".csv").
- `%t` — Timestamp (in microseconds).
diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md
index 5804ad8545b..fe6e8e15b0c 100644
--- a/docs/en/operations/storing-data.md
+++ b/docs/en/operations/storing-data.md
@@ -184,13 +184,15 @@ These settings should be defined in the disk configuration section.
- `enable_filesystem_query_cache_limit` - allow to limit the size of cache which is downloaded within each query (depends on user setting `max_query_cache_size`). Default: `false`.
-- `enable_cache_hits_threshold` - number which defines how many times some data needs to be read before it will be cached. Default: `0`, e.g. the data is cached at the first attempt to read it.
+- `enable_cache_hits_threshold` - number which defines how many times some data needs to be read before it will be cached. Default: `false`. This threshold can be defined by `cache_hits_threshold`. Default: `0`, e.g. the data is cached at the first attempt to read it.
+
+- `enable_bypass_cache_with_threshold` - allows to skip cache completely in case the requested read range exceeds the threshold. Default: `false`. This threshold can be defined by `bypass_cache_threashold`. Default: `268435456` (`256Mi`).
- `do_not_evict_index_and_mark_files` - do not evict small frequently used files according to cache policy. Default: `false`. This setting was added in version 22.8. If you used filesystem cache before this version, then it will not work on versions starting from 22.8 if this setting is set to `true`. If you want to use this setting, clear old cache created before version 22.8 before upgrading.
-- `max_file_segment_size` - a maximum size of a single cache file in bytes or in readable format (`ki, Mi, Gi, etc`, example `10Gi`). Default: `104857600` (`100Mi`).
+- `max_file_segment_size` - a maximum size of a single cache file in bytes or in readable format (`ki, Mi, Gi, etc`, example `10Gi`). Default: `8388608` (`8Mi`).
-- `max_elements` - a limit for a number of cache files. Default: `1048576`.
+- `max_elements` - a limit for a number of cache files. Default: `10000000`.
File Cache **query/profile settings**:
diff --git a/docs/en/operations/system-tables/asynchronous_metric_log.md b/docs/en/operations/system-tables/asynchronous_metric_log.md
index 4290799b6bc..efe57a202d8 100644
--- a/docs/en/operations/system-tables/asynchronous_metric_log.md
+++ b/docs/en/operations/system-tables/asynchronous_metric_log.md
@@ -9,7 +9,6 @@ Columns:
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
-- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds resolution.
- `name` ([String](../../sql-reference/data-types/string.md)) — Metric name.
- `value` ([Float64](../../sql-reference/data-types/float.md)) — Metric value.
@@ -20,18 +19,18 @@ SELECT * FROM system.asynchronous_metric_log LIMIT 10
```
``` text
-┌─event_date─┬──────────event_time─┬────event_time_microseconds─┬─name─────────────────────────────────────┬─────value─┐
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ CPUFrequencyMHz_0 │ 2120.9 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pmuzzy │ 743 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pdirty │ 26288 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.run_intervals │ 0 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.num_runs │ 0 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.retained │ 60694528 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.mapped │ 303161344 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.resident │ 260931584 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.metadata │ 12079488 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.allocated │ 133756128 │
-└────────────┴─────────────────────┴────────────────────────────┴──────────────────────────────────────────┴───────────┘
+┌─event_date─┬──────────event_time─┬─name─────────────────────────────────────┬─────value─┐
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ CPUFrequencyMHz_0 │ 2120.9 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.arenas.all.pmuzzy │ 743 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.arenas.all.pdirty │ 26288 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.background_thread.run_intervals │ 0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.background_thread.num_runs │ 0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.retained │ 60694528 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.mapped │ 303161344 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.resident │ 260931584 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.metadata │ 12079488 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.allocated │ 133756128 │
+└────────────┴─────────────────────┴──────────────────────────────────────────┴───────────┘
```
**See Also**
diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md
index 508419783ef..1b720098fc7 100644
--- a/docs/en/operations/system-tables/index.md
+++ b/docs/en/operations/system-tables/index.md
@@ -13,6 +13,7 @@ System tables provide information about:
- Server states, processes, and environment.
- Server’s internal processes.
+- Options used when the ClickHouse binary was built.
System tables:
diff --git a/docs/en/operations/system-tables/jemalloc_bins.md b/docs/en/operations/system-tables/jemalloc_bins.md
new file mode 100644
index 00000000000..06d9ba57dfc
--- /dev/null
+++ b/docs/en/operations/system-tables/jemalloc_bins.md
@@ -0,0 +1,45 @@
+---
+slug: /en/operations/system-tables/jemalloc_bins
+---
+# jemalloc_bins
+
+Contains information about memory allocations done via jemalloc allocator in different size classes (bins) aggregated from all arenas.
+These statistics might not be absolutely accurate because of thread local caching in jemalloc.
+
+Columns:
+
+- `index` (UInt64) — Index of the bin ordered by size
+- `large` (Bool) — True for large allocations and False for small
+- `size` (UInt64) — Size of allocations in this bin
+- `allocations` (UInt64) — Number of allocations
+- `deallocations` (UInt64) — Number of deallocations
+
+**Example**
+
+Find the sizes of allocations that contributed the most to the current overall memory usage.
+
+``` sql
+SELECT
+ *,
+ allocations - deallocations AS active_allocations,
+ size * active_allocations AS allocated_bytes
+FROM system.jemalloc_bins
+WHERE allocated_bytes > 0
+ORDER BY allocated_bytes DESC
+LIMIT 10
+```
+
+``` text
+┌─index─┬─large─┬─────size─┬─allocactions─┬─deallocations─┬─active_allocations─┬─allocated_bytes─┐
+│ 82 │ 1 │ 50331648 │ 1 │ 0 │ 1 │ 50331648 │
+│ 10 │ 0 │ 192 │ 512336 │ 370710 │ 141626 │ 27192192 │
+│ 69 │ 1 │ 5242880 │ 6 │ 2 │ 4 │ 20971520 │
+│ 3 │ 0 │ 48 │ 16938224 │ 16559484 │ 378740 │ 18179520 │
+│ 28 │ 0 │ 4096 │ 122924 │ 119142 │ 3782 │ 15491072 │
+│ 61 │ 1 │ 1310720 │ 44569 │ 44558 │ 11 │ 14417920 │
+│ 39 │ 1 │ 28672 │ 1285 │ 913 │ 372 │ 10665984 │
+│ 4 │ 0 │ 64 │ 2837225 │ 2680568 │ 156657 │ 10026048 │
+│ 6 │ 0 │ 96 │ 2617803 │ 2531435 │ 86368 │ 8291328 │
+│ 36 │ 1 │ 16384 │ 22431 │ 21970 │ 461 │ 7553024 │
+└───────┴───────┴──────────┴──────────────┴───────────────┴────────────────────┴─────────────────┘
+```
diff --git a/docs/en/operations/system-tables/merge_tree_settings.md b/docs/en/operations/system-tables/merge_tree_settings.md
index d8539908bf7..557835ce3b6 100644
--- a/docs/en/operations/system-tables/merge_tree_settings.md
+++ b/docs/en/operations/system-tables/merge_tree_settings.md
@@ -7,11 +7,17 @@ Contains information about settings for `MergeTree` tables.
Columns:
-- `name` (String) — Setting name.
-- `value` (String) — Setting value.
-- `description` (String) — Setting description.
-- `type` (String) — Setting type (implementation specific string value).
-- `changed` (UInt8) — Whether the setting was explicitly defined in the config or explicitly changed.
+- `name` ([String](../../sql-reference/data-types/string.md)) — Setting name.
+- `value` ([String](../../sql-reference/data-types/string.md)) — Setting value.
+- `changed` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Whether the setting was explicitly defined in the config or explicitly changed.
+- `description` ([String](../../sql-reference/data-types/string.md)) — Setting description.
+- `min` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Minimum value of the setting, if any is set via [constraints](../../operations/settings/constraints-on-settings.md#constraints-on-settings). If the setting has no minimum value, contains [NULL](../../sql-reference/syntax.md#null-literal).
+- `max` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Maximum value of the setting, if any is set via [constraints](../../operations/settings/constraints-on-settings.md#constraints-on-settings). If the setting has no maximum value, contains [NULL](../../sql-reference/syntax.md#null-literal).
+- `readonly` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether the current user can change the setting:
+ - `0` — Current user can change the setting.
+ - `1` — Current user can’t change the setting.
+- `type` ([String](../../sql-reference/data-types/string.md)) — Setting type (implementation specific string value).
+- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) _ Shows whether a setting is obsolete.
**Example**
```sql
@@ -21,35 +27,51 @@ SELECT * FROM system.merge_tree_settings LIMIT 4 FORMAT Vertical;
```response
Row 1:
──────
+name: min_compress_block_size
+value: 0
+changed: 0
+description: When granule is written, compress the data in buffer if the size of pending uncompressed data is larger or equal than the specified threshold. If this setting is not set, the corresponding global setting is used.
+min: ____
+max: ____
+readonly: 0
+type: UInt64
+is_obsolete: 0
+
+Row 2:
+──────
+name: max_compress_block_size
+value: 0
+changed: 0
+description: Compress the pending uncompressed data in buffer if its size is larger or equal than the specified threshold. Block of data will be compressed even if the current granule is not finished. If this setting is not set, the corresponding global setting is used.
+min: ____
+max: ____
+readonly: 0
+type: UInt64
+is_obsolete: 0
+
+Row 3:
+──────
name: index_granularity
value: 8192
changed: 0
description: How many rows correspond to one primary key value.
-type: SettingUInt64
-
-Row 2:
-──────
-name: min_bytes_for_wide_part
-value: 0
-changed: 0
-description: Minimal uncompressed size in bytes to create part in wide format instead of compact
-type: SettingUInt64
-
-Row 3:
-──────
-name: min_rows_for_wide_part
-value: 0
-changed: 0
-description: Minimal number of rows to create part in wide format instead of compact
-type: SettingUInt64
+min: ____
+max: ____
+readonly: 0
+type: UInt64
+is_obsolete: 0
Row 4:
──────
-name: merge_max_block_size
-value: 8192
+name: max_digestion_size_per_segment
+value: 268435456
changed: 0
-description: How many rows in blocks should be formed for merge operations.
-type: SettingUInt64
+description: Max number of bytes to digest per segment to build GIN index.
+min: ____
+max: ____
+readonly: 0
+type: UInt64
+is_obsolete: 0
-4 rows in set. Elapsed: 0.001 sec.
+4 rows in set. Elapsed: 0.009 sec.
```
diff --git a/docs/en/operations/system-tables/parts.md b/docs/en/operations/system-tables/parts.md
index 9159d1e9284..8113b850a38 100644
--- a/docs/en/operations/system-tables/parts.md
+++ b/docs/en/operations/system-tables/parts.md
@@ -27,7 +27,7 @@ Columns:
Data storing format is controlled by the `min_bytes_for_wide_part` and `min_rows_for_wide_part` settings of the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table.
- - `active` ([UInt8](../../sql-reference/data-types/int-uint.md)) – Flag that indicates whether the data part is active. If a data part is active, it’s used in a table. Otherwise, it’s deleted. Inactive data parts remain after merging.
+- `active` ([UInt8](../../sql-reference/data-types/int-uint.md)) – Flag that indicates whether the data part is active. If a data part is active, it’s used in a table. Otherwise, it’s deleted. Inactive data parts remain after merging.
- `marks` ([UInt64](../../sql-reference/data-types/int-uint.md)) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` by the index granularity (usually 8192) (this hint does not work for adaptive granularity).
@@ -39,6 +39,8 @@ Columns:
- `data_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – Total size of uncompressed data in the data part. All the auxiliary files (for example, files with marks) are not included.
+- `primary_key_size` ([UInt64](../../sql-reference/data-types/int-uint.md)) – The amount of memory (in bytes) used by primary key values in the primary.idx/cidx file on disk.
+
- `marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – The size of the file with marks.
- `secondary_indices_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – Total size of compressed data for secondary indices in the data part. All the auxiliary files (for example, files with marks) are not included.
diff --git a/docs/en/operations/system-tables/query_log.md b/docs/en/operations/system-tables/query_log.md
index 71e1452cef1..b9fdd19c643 100644
--- a/docs/en/operations/system-tables/query_log.md
+++ b/docs/en/operations/system-tables/query_log.md
@@ -71,11 +71,11 @@ Columns:
- 0 — Query was initiated by another query as part of distributed query execution.
- `user` ([String](../../sql-reference/data-types/string.md)) — Name of the user who initiated the current query.
- `query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the query.
-- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query.
+- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP address that was used to make the query.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to make the query.
- `initial_user` ([String](../../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution).
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution).
-- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from.
+- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP address that the parent query was launched from.
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to make the parent query.
- `initial_query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Initial query starting time (for distributed query execution).
- `initial_query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Initial query starting time with microseconds precision (for distributed query execution).
diff --git a/docs/en/operations/system-tables/query_thread_log.md b/docs/en/operations/system-tables/query_thread_log.md
index cdd23bb15db..a6d5632ade9 100644
--- a/docs/en/operations/system-tables/query_thread_log.md
+++ b/docs/en/operations/system-tables/query_thread_log.md
@@ -40,11 +40,11 @@ Columns:
- 0 — Query was initiated by another query for distributed query execution.
- `user` ([String](../../sql-reference/data-types/string.md)) — Name of the user who initiated the current query.
- `query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the query.
-- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query.
+- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP address that was used to make the query.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the query.
- `initial_user` ([String](../../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution).
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution).
-- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from.
+- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP address that the parent query was launched from.
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the parent query.
- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Interface that the query was initiated from. Possible values:
- 1 — TCP.
diff --git a/docs/en/operations/system-tables/server_settings.md b/docs/en/operations/system-tables/server_settings.md
index 3085b1acaf4..df482261ae8 100644
--- a/docs/en/operations/system-tables/server_settings.md
+++ b/docs/en/operations/system-tables/server_settings.md
@@ -14,6 +14,7 @@ Columns:
- `changed` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether a setting was specified in `config.xml`
- `description` ([String](../../sql-reference/data-types/string.md)) — Short server setting description.
- `type` ([String](../../sql-reference/data-types/string.md)) — Server setting value type.
+- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) _ Shows whether a setting is obsolete.
**Example**
@@ -26,14 +27,22 @@ WHERE name LIKE '%thread_pool%'
```
``` text
-┌─name─────────────────────────┬─value─┬─default─┬─changed─┬─description─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─type───┐
-│ max_thread_pool_size │ 5000 │ 10000 │ 1 │ The maximum number of threads that could be allocated from the OS and used for query execution and background operations. │ UInt64 │
-│ max_thread_pool_free_size │ 1000 │ 1000 │ 0 │ The maximum number of threads that will always stay in a global thread pool once allocated and remain idle in case of insufficient number of tasks. │ UInt64 │
-│ thread_pool_queue_size │ 10000 │ 10000 │ 0 │ The maximum number of tasks that will be placed in a queue and wait for execution. │ UInt64 │
-│ max_io_thread_pool_size │ 100 │ 100 │ 0 │ The maximum number of threads that would be used for IO operations │ UInt64 │
-│ max_io_thread_pool_free_size │ 0 │ 0 │ 0 │ Max free size for IO thread pool. │ UInt64 │
-│ io_thread_pool_queue_size │ 10000 │ 10000 │ 0 │ Queue size for IO thread pool. │ UInt64 │
-└──────────────────────────────┴───────┴─────────┴─────────┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴────────┘
+┌─name────────────────────────────────────────_─value─_─default─_─changed─_─description──────────────────────────────────────────────────────────────────────────────────────────────────────
+───────────────────────────────────_─type───_─is_obsolete─┐
+│ max_thread_pool_size │ 10000 │ 10000 │ 1 │ The maximum number of threads that could be allocated from the OS and used for query execution and background operations. │ UInt64 │ 0 │
+│ max_thread_pool_free_size │ 1000 │ 1000 │ 0 │ The maximum number of threads that will always stay in a global thread pool once allocated and remain idle in case of insufficient number of tasks. │ UInt64 │ 0 │
+│ thread_pool_queue_size │ 10000 │ 10000 │ 0 │ The maximum number of tasks that will be placed in a queue and wait for execution. │ UInt64 │ 0 │
+│ max_io_thread_pool_size │ 100 │ 100 │ 0 │ The maximum number of threads that would be used for IO operations │ UInt64 │ 0 │
+│ max_io_thread_pool_free_size │ 0 │ 0 │ 0 │ Max free size for IO thread pool. │ UInt64 │ 0 │
+│ io_thread_pool_queue_size │ 10000 │ 10000 │ 0 │ Queue size for IO thread pool. │ UInt64 │ 0 │
+│ max_active_parts_loading_thread_pool_size │ 64 │ 64 │ 0 │ The number of threads to load active set of data parts (Active ones) at startup. │ UInt64 │ 0 │
+│ max_outdated_parts_loading_thread_pool_size │ 32 │ 32 │ 0 │ The number of threads to load inactive set of data parts (Outdated ones) at startup. │ UInt64 │ 0 │
+│ max_parts_cleaning_thread_pool_size │ 128 │ 128 │ 0 │ The number of threads for concurrent removal of inactive data parts. │ UInt64 │ 0 │
+│ max_backups_io_thread_pool_size │ 1000 │ 1000 │ 0 │ The maximum number of threads that would be used for IO operations for BACKUP queries │ UInt64 │ 0 │
+│ max_backups_io_thread_pool_free_size │ 0 │ 0 │ 0 │ Max free size for backups IO thread pool. │ UInt64 │ 0 │
+│ backups_io_thread_pool_queue_size │ 0 │ 0 │ 0 │ Queue size for backups IO thread pool. │ UInt64 │ 0 │
+└─────────────────────────────────────────────┴───────┴─────────┴─────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────
+───────────────────────────────────┴────────┴─────────────┘
```
Using of `WHERE changed` can be useful, for example, when you want to check
diff --git a/docs/en/operations/system-tables/session_log.md b/docs/en/operations/system-tables/session_log.md
index 661d34677e4..5b1a2b2a489 100644
--- a/docs/en/operations/system-tables/session_log.md
+++ b/docs/en/operations/system-tables/session_log.md
@@ -28,7 +28,7 @@ Columns:
- `profiles` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — The list of profiles set for all roles and/or users.
- `roles` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — The list of roles to which the profile is applied.
- `settings` ([Array](../../sql-reference/data-types/array.md)([Tuple](../../sql-reference/data-types/tuple.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md), [String](../../sql-reference/data-types/string.md)))) — Settings that were changed when the client logged in/out.
-- `client_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — The IP address that was used to log in/out.
+- `client_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — The IP address that was used to log in/out.
- `client_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to log in/out.
- `interface` ([Enum8](../../sql-reference/data-types/enum.md)) — The interface from which the login was initiated. Possible values:
- `TCP`
diff --git a/docs/en/operations/system-tables/settings.md b/docs/en/operations/system-tables/settings.md
index afae45077cc..7dd2345a2d0 100644
--- a/docs/en/operations/system-tables/settings.md
+++ b/docs/en/operations/system-tables/settings.md
@@ -17,6 +17,7 @@ Columns:
- `0` — Current user can change the setting.
- `1` — Current user can’t change the setting.
- `default` ([String](../../sql-reference/data-types/string.md)) — Setting default value.
+- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) _ Shows whether a setting is obsolete.
**Example**
@@ -29,11 +30,14 @@ WHERE name LIKE '%min_i%'
```
``` text
-┌─name────────────────────────────────────────┬─value─────┬─changed─┬─description───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─min──┬─max──┬─readonly─┐
-│ min_insert_block_size_rows │ 1048576 │ 0 │ Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │
-│ min_insert_block_size_bytes │ 268435456 │ 0 │ Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │
-│ read_backoff_min_interval_between_events_ms │ 1000 │ 0 │ Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time. │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │
-└─────────────────────────────────────────────┴───────────┴─────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴──────┴──────┴──────────┘
+┌─name───────────────────────────────────────────────_─value─────_─changed─_─description───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────_─min──_─max──_─readonly─_─type─────────_─default───_─alias_for─_─is_obsolete─┐
+│ min_insert_block_size_rows │ 1048449 │ 0 │ Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough. │ ____ │ ____ │ 0 │ UInt64 │ 1048449 │ │ 0 │
+│ min_insert_block_size_bytes │ 268402944 │ 0 │ Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough. │ ____ │ ____ │ 0 │ UInt64 │ 268402944 │ │ 0 │
+│ min_insert_block_size_rows_for_materialized_views │ 0 │ 0 │ Like min_insert_block_size_rows, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_rows) │ ____ │ ____ │ 0 │ UInt64 │ 0 │ │ 0 │
+│ min_insert_block_size_bytes_for_materialized_views │ 0 │ 0 │ Like min_insert_block_size_bytes, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_bytes) │ ____ │ ____ │ 0 │ UInt64 │ 0 │ │ 0 │
+│ read_backoff_min_interval_between_events_ms │ 1000 │ 0 │ Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time. │ ____ │ ____ │ 0 │ Milliseconds │ 1000 │ │ 0 │
+└────────────────────────────────────────────────────┴───────────┴─────────┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────
+──────────────────────────────────────────────────────┴──────┴──────┴──────────┴──────────────┴───────────┴───────────┴─────────────┘
```
Using of `WHERE changed` can be useful, for example, when you want to check:
diff --git a/docs/en/operations/system-tables/zookeeper_connection.md b/docs/en/operations/system-tables/zookeeper_connection.md
index 9438cda1808..2f0ed303ce3 100644
--- a/docs/en/operations/system-tables/zookeeper_connection.md
+++ b/docs/en/operations/system-tables/zookeeper_connection.md
@@ -11,7 +11,8 @@ Columns:
- `host` ([String](../../sql-reference/data-types/string.md)) — The hostname/IP of the ZooKeeper node that ClickHouse connected to.
- `port` ([String](../../sql-reference/data-types/string.md)) — The port of the ZooKeeper node that ClickHouse connected to.
- `index` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The index of the ZooKeeper node that ClickHouse connected to. The index is from ZooKeeper config.
-- `connected_time` ([String](../../sql-reference/data-types/string.md)) — When the connection was established
+- `connected_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — When the connection was established
+- `session_uptime_elapsed_seconds` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Seconds elapsed since the connection was established
- `is_expired` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the current connection expired.
- `keeper_api_version` ([String](../../sql-reference/data-types/string.md)) — Keeper API version.
- `client_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Session id of the connection.
@@ -23,7 +24,7 @@ SELECT * FROM system.zookeeper_connection;
```
``` text
-┌─name──────────────┬─host─────────┬─port─┬─index─┬──────connected_time─┬─is_expired─┬─keeper_api_version─┬──────────client_id─┐
-│ default_zookeeper │ 127.0.0.1 │ 2181 │ 0 │ 2023-05-19 14:30:16 │ 0 │ 0 │ 216349144108826660 │
-└───────────────────┴──────────────┴──────┴───────┴─────────────────────┴────────────┴────────────────────┴────────────────────┘
+┌─name────┬─host──────┬─port─┬─index─┬──────connected_time─┬─session_uptime_elapsed_seconds─┬─is_expired─┬─keeper_api_version─┬─client_id─┐
+│ default │ 127.0.0.1 │ 9181 │ 0 │ 2023-06-15 14:36:01 │ 3058 │ 0 │ 3 │ 5 │
+└─────────┴───────────┴──────┴───────┴─────────────────────┴────────────────────────────────┴────────────┴────────────────────┴───────────┘
```
diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md
index b7cc4e22cd6..dce5be29f62 100644
--- a/docs/en/operations/system-tables/zookeeper_log.md
+++ b/docs/en/operations/system-tables/zookeeper_log.md
@@ -15,7 +15,7 @@ Columns with request parameters:
- `Finalize` — The connection is lost, no response was received.
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — The date when the event happened.
- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — The date and time when the event happened.
-- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address of ZooKeeper server that was used to make the request.
+- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP address of ZooKeeper server that was used to make the request.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The port of ZooKeeper server that was used to make the request.
- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — The session ID that the ZooKeeper server sets for each connection.
- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. This is usually a sequential request number. It is the same for the request row and the paired `response`/`finalize` row.
diff --git a/docs/en/sql-reference/aggregate-functions/combinators.md b/docs/en/sql-reference/aggregate-functions/combinators.md
index fd693430064..18ff5073e3f 100644
--- a/docs/en/sql-reference/aggregate-functions/combinators.md
+++ b/docs/en/sql-reference/aggregate-functions/combinators.md
@@ -97,6 +97,10 @@ Result:
If you apply this combinator, the aggregate function does not return the resulting value (such as the number of unique values for the [uniq](../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function), but an intermediate state of the aggregation (for `uniq`, this is the hash table for calculating the number of unique values). This is an `AggregateFunction(...)` that can be used for further processing or stored in a table to finish aggregating later.
+:::note
+Please notice, that -MapState is not an invariant for the same data due to the fact that order of data in intermediate state can change, though it doesn't impact ingestion of this data.
+:::
+
To work with these states, use:
- [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) table engine.
@@ -296,7 +300,7 @@ SELECT groupArrayResample(30, 75, 30)(name, age) FROM people
Consider the results.
-`Jonh` is out of the sample because he’s too young. Other people are distributed according to the specified age intervals.
+`John` is out of the sample because he’s too young. Other people are distributed according to the specified age intervals.
Now let’s count the total number of people and their average wage in the specified age intervals.
diff --git a/docs/en/sql-reference/aggregate-functions/reference/any.md b/docs/en/sql-reference/aggregate-functions/reference/any.md
index db19f524b31..f79fe66c05d 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/any.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/any.md
@@ -12,3 +12,5 @@ To get a determinate result, you can use the ‘min’ or ‘max’ function ins
In some cases, you can rely on the order of execution. This applies to cases when SELECT comes from a subquery that uses ORDER BY.
When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function.
+
+- Alias: `any_value`
diff --git a/docs/en/sql-reference/aggregate-functions/reference/arrayconcatagg.md b/docs/en/sql-reference/aggregate-functions/reference/arrayconcatagg.md
new file mode 100644
index 00000000000..3c71129bdb5
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/arrayconcatagg.md
@@ -0,0 +1,32 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/array_concat_agg
+sidebar_position: 110
+---
+
+# array_concat_agg
+- Alias of `groupArrayArray`. The function is case insensitive.
+
+**Example**
+
+```text
+SELECT *
+FROM t
+
+┌─a───────┐
+│ [1,2,3] │
+│ [4,5] │
+│ [6] │
+└─────────┘
+
+```
+
+Query:
+
+```sql
+SELECT array_concat_agg(a) AS a
+FROM t
+
+┌─a─────────────┐
+│ [1,2,3,4,5,6] │
+└───────────────┘
+```
diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparray.md b/docs/en/sql-reference/aggregate-functions/reference/grouparray.md
index 18048fa4f71..ad678443df6 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/grouparray.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/grouparray.md
@@ -44,3 +44,5 @@ Result:
```
The groupArray function will remove ᴺᵁᴸᴸ value based on the above results.
+
+- Alias: `array_agg`.
diff --git a/docs/en/sql-reference/aggregate-functions/reference/index.md b/docs/en/sql-reference/aggregate-functions/reference/index.md
index 17ef494e9ad..6c56aefd51d 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/index.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/index.md
@@ -19,8 +19,19 @@ Standard aggregate functions:
- [stddevSamp](/docs/en/sql-reference/aggregate-functions/reference/stddevsamp.md)
- [varPop](/docs/en/sql-reference/aggregate-functions/reference/varpop.md)
- [varSamp](/docs/en/sql-reference/aggregate-functions/reference/varsamp.md)
+- [corr](./corr.md)
- [covarPop](/docs/en/sql-reference/aggregate-functions/reference/covarpop.md)
- [covarSamp](/docs/en/sql-reference/aggregate-functions/reference/covarsamp.md)
+- [entropy](./entropy.md)
+- [exponentialMovingAverage](./exponentialmovingaverage.md)
+- [intervalLengthSum](./intervalLengthSum.md)
+- [kolmogorovSmirnovTest](./kolmogorovsmirnovtest.md)
+- [mannwhitneyutest](./mannwhitneyutest.md)
+- [median](./median.md)
+- [rankCorr](./rankCorr.md)
+- [sumKahan](./sumkahan.md)
+- [studentTTest](./studentttest.md)
+- [welchTTest](./welchttest.md)
ClickHouse-specific aggregate functions:
@@ -34,12 +45,15 @@ ClickHouse-specific aggregate functions:
- [avgWeighted](/docs/en/sql-reference/aggregate-functions/reference/avgweighted.md)
- [topK](/docs/en/sql-reference/aggregate-functions/reference/topk.md)
- [topKWeighted](/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md)
+- [deltaSum](./deltasum.md)
+- [deltaSumTimestamp](./deltasumtimestamp.md)
- [groupArray](/docs/en/sql-reference/aggregate-functions/reference/grouparray.md)
- [groupArrayLast](/docs/en/sql-reference/aggregate-functions/reference/grouparraylast.md)
- [groupUniqArray](/docs/en/sql-reference/aggregate-functions/reference/groupuniqarray.md)
- [groupArrayInsertAt](/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat.md)
- [groupArrayMovingAvg](/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingavg.md)
- [groupArrayMovingSum](/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingsum.md)
+- [groupArraySample](./grouparraysample.md)
- [groupBitAnd](/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md)
- [groupBitOr](/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md)
- [groupBitXor](/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md)
@@ -84,3 +98,9 @@ ClickHouse-specific aggregate functions:
- [theilsU](./theilsu.md)
- [maxIntersections](./maxintersections.md)
- [maxIntersectionsPosition](./maxintersectionsposition.md)
+- [meanZTest](./meanztest.md)
+- [quantileGK](./quantileGK.md)
+- [quantileInterpolatedWeighted](./quantileinterpolatedweighted.md)
+- [sparkBar](./sparkbar.md)
+- [sumCount](./sumcount.md)
+
diff --git a/docs/en/sql-reference/data-types/datetime.md b/docs/en/sql-reference/data-types/datetime.md
index 0da273e01ad..fe279edb709 100644
--- a/docs/en/sql-reference/data-types/datetime.md
+++ b/docs/en/sql-reference/data-types/datetime.md
@@ -143,5 +143,6 @@ Time shifts for multiple days. Some pacific islands changed their timezone offse
- [The `date_time_input_format` setting](../../operations/settings/settings.md#settings-date_time_input_format)
- [The `date_time_output_format` setting](../../operations/settings/settings.md#settings-date_time_output_format)
- [The `timezone` server configuration parameter](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
+- [The `session_timezone` setting](../../operations/settings/settings.md#session_timezone)
- [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime)
- [The `Date` data type](../../sql-reference/data-types/date.md)
diff --git a/docs/en/sql-reference/data-types/datetime64.md b/docs/en/sql-reference/data-types/datetime64.md
index 793691850b1..3b80e8b1a8b 100644
--- a/docs/en/sql-reference/data-types/datetime64.md
+++ b/docs/en/sql-reference/data-types/datetime64.md
@@ -119,6 +119,7 @@ FROM dt;
- [The `date_time_input_format` setting](../../operations/settings/settings-formats.md#date_time_input_format)
- [The `date_time_output_format` setting](../../operations/settings/settings-formats.md#date_time_output_format)
- [The `timezone` server configuration parameter](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
+- [The `session_timezone` setting](../../operations/settings/settings.md#session_timezone)
- [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-for-working-with-dates-and-times)
- [`Date` data type](../../sql-reference/data-types/date.md)
- [`DateTime` data type](../../sql-reference/data-types/datetime.md)
diff --git a/docs/en/sql-reference/data-types/decimal.md b/docs/en/sql-reference/data-types/decimal.md
index 8df8b2519e3..bba5ea74ebe 100644
--- a/docs/en/sql-reference/data-types/decimal.md
+++ b/docs/en/sql-reference/data-types/decimal.md
@@ -32,7 +32,7 @@ For example, Decimal32(4) can contain numbers from -99999.9999 to 99999.9999 wit
Internally data is represented as normal signed integers with respective bit width. Real value ranges that can be stored in memory are a bit larger than specified above, which are checked only on conversion from a string.
-Because modern CPUs do not support 128-bit integers natively, operations on Decimal128 are emulated. Because of this Decimal128 works significantly slower than Decimal32/Decimal64.
+Because modern CPUs do not support 128-bit and 256-bit integers natively, operations on Decimal128 and Decimal256 are emulated. Thus, Decimal128 and Decimal256 work significantly slower than Decimal32/Decimal64.
## Operations and Result Type
@@ -59,6 +59,10 @@ Some functions on Decimal return result as Float64 (for example, var or stddev).
During calculations on Decimal, integer overflows might happen. Excessive digits in a fraction are discarded (not rounded). Excessive digits in integer part will lead to an exception.
+:::warning
+Overflow check is not implemented for Decimal128 and Decimal256. In case of overflow incorrect result is returned, no exception is thrown.
+:::
+
``` sql
SELECT toDecimal32(2, 4) AS x, x / 3
```
diff --git a/docs/en/sql-reference/data-types/index.md b/docs/en/sql-reference/data-types/index.md
index 508307a0543..ffd063590fa 100644
--- a/docs/en/sql-reference/data-types/index.md
+++ b/docs/en/sql-reference/data-types/index.md
@@ -28,6 +28,6 @@ ClickHouse data types include:
- **Nested data structures**: A [`Nested` data structure](./nested-data-structures/index.md) is like a table inside a cell
- **Tuples**: A [`Tuple` of elements](./tuple.md), each having an individual type.
- **Nullable**: [`Nullable`](./nullable.md) allows you to store a value as `NULL` when a value is "missing" (instead of the column settings its default value for the data type)
-- **IP addresses**: use [`IPv4`](./domains/ipv4.md) and [`IPv6`](./domains/ipv6.md) to efficiently store IP addresses
+- **IP addresses**: use [`IPv4`](./ipv4.md) and [`IPv6`](./ipv6.md) to efficiently store IP addresses
- **Geo types**: for [geographical data](./geo.md), including `Point`, `Ring`, `Polygon` and `MultiPolygon`
- **Special data types**: including [`Expression`](./special-data-types/expression.md), [`Set`](./special-data-types/set.md), [`Nothing`](./special-data-types/nothing.md) and [`Interval`](./special-data-types/interval.md)
diff --git a/docs/en/sql-reference/data-types/domains/ipv4.md b/docs/en/sql-reference/data-types/ipv4.md
similarity index 60%
rename from docs/en/sql-reference/data-types/domains/ipv4.md
rename to docs/en/sql-reference/data-types/ipv4.md
index b34814211fc..288806f47b3 100644
--- a/docs/en/sql-reference/data-types/domains/ipv4.md
+++ b/docs/en/sql-reference/data-types/ipv4.md
@@ -1,12 +1,12 @@
---
-slug: /en/sql-reference/data-types/domains/ipv4
+slug: /en/sql-reference/data-types/ipv4
sidebar_position: 59
sidebar_label: IPv4
---
## IPv4
-`IPv4` is a domain based on `UInt32` type and serves as a typed replacement for storing IPv4 values. It provides compact storage with the human-friendly input-output format and column type information on inspection.
+IPv4 addresses. Stored in 4 bytes as UInt32.
### Basic Usage
@@ -57,25 +57,6 @@ SELECT toTypeName(from), hex(from) FROM hits LIMIT 1;
└──────────────────┴───────────┘
```
-Domain values are not implicitly convertible to types other than `UInt32`.
-If you want to convert `IPv4` value to a string, you have to do that explicitly with `IPv4NumToString()` function:
+**See Also**
-``` sql
-SELECT toTypeName(s), IPv4NumToString(from) as s FROM hits LIMIT 1;
-```
-
- ┌─toTypeName(IPv4NumToString(from))─┬─s──────────────┐
- │ String │ 183.247.232.58 │
- └───────────────────────────────────┴────────────────┘
-
-Or cast to a `UInt32` value:
-
-``` sql
-SELECT toTypeName(i), CAST(from as UInt32) as i FROM hits LIMIT 1;
-```
-
-``` text
-┌─toTypeName(CAST(from, 'UInt32'))─┬──────────i─┐
-│ UInt32 │ 3086477370 │
-└──────────────────────────────────┴────────────┘
-```
+- [Functions for Working with IPv4 and IPv6 Addresses](../functions/ip-address-functions.md)
diff --git a/docs/en/sql-reference/data-types/domains/ipv6.md b/docs/en/sql-reference/data-types/ipv6.md
similarity index 61%
rename from docs/en/sql-reference/data-types/domains/ipv6.md
rename to docs/en/sql-reference/data-types/ipv6.md
index dcb22e3cb6d..97959308b58 100644
--- a/docs/en/sql-reference/data-types/domains/ipv6.md
+++ b/docs/en/sql-reference/data-types/ipv6.md
@@ -1,12 +1,12 @@
---
-slug: /en/sql-reference/data-types/domains/ipv6
+slug: /en/sql-reference/data-types/ipv6
sidebar_position: 60
sidebar_label: IPv6
---
## IPv6
-`IPv6` is a domain based on `FixedString(16)` type and serves as a typed replacement for storing IPv6 values. It provides compact storage with the human-friendly input-output format and column type information on inspection.
+IPv6 addresses. Stored in 16 bytes as UInt128 big-endian.
### Basic Usage
@@ -57,27 +57,6 @@ SELECT toTypeName(from), hex(from) FROM hits LIMIT 1;
└──────────────────┴──────────────────────────────────┘
```
-Domain values are not implicitly convertible to types other than `FixedString(16)`.
-If you want to convert `IPv6` value to a string, you have to do that explicitly with `IPv6NumToString()` function:
+**See Also**
-``` sql
-SELECT toTypeName(s), IPv6NumToString(from) as s FROM hits LIMIT 1;
-```
-
-``` text
-┌─toTypeName(IPv6NumToString(from))─┬─s─────────────────────────────┐
-│ String │ 2001:44c8:129:2632:33:0:252:2 │
-└───────────────────────────────────┴───────────────────────────────┘
-```
-
-Or cast to a `FixedString(16)` value:
-
-``` sql
-SELECT toTypeName(i), CAST(from as FixedString(16)) as i FROM hits LIMIT 1;
-```
-
-``` text
-┌─toTypeName(CAST(from, 'FixedString(16)'))─┬─i───────┐
-│ FixedString(16) │ ��� │
-└───────────────────────────────────────────┴─────────┘
-```
+- [Functions for Working with IPv4 and IPv6 Addresses](../functions/ip-address-functions.md)
diff --git a/docs/en/sql-reference/functions/arithmetic-functions.md b/docs/en/sql-reference/functions/arithmetic-functions.md
index 64fae0e82f0..69f1816b7df 100644
--- a/docs/en/sql-reference/functions/arithmetic-functions.md
+++ b/docs/en/sql-reference/functions/arithmetic-functions.md
@@ -6,9 +6,20 @@ sidebar_label: Arithmetic
# Arithmetic Functions
-The result type of all arithmetic functions is the smallest type which can represent all possible results. Size promotion happens for integers up to 32 bit, e.g. `UInt8 + UInt16 = UInt32`. If one of the inters has 64 or more bits, the result is of the same type as the bigger of the input integers, e.g. `UInt16 + UInt128 = UInt128`. While this introduces a risk of overflows around the value range boundary, it ensures that calculations are performed quickly using the maximum native integer width of 64 bit.
+Arithmetic functions work for any two operands of type `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64`, `Float32`, or `Float64`.
-The result of addition or multiplication of two integers is unsigned unless one of the integers is signed.
+Before performing the operation, both operands are casted to the result type. The result type is determined as follows (unless specified
+differently in the function documentation below):
+- If both operands are up to 32 bits wide, the size of the result type will be the size of the next bigger type following the bigger of the
+ two operands (integer size promotion). For example, `UInt8 + UInt16 = UInt32` or `Float32 * Float32 = Float64`.
+- If one of the operands has 64 or more bits, the size of the result type will be the same size as the bigger of the two operands. For
+ example, `UInt32 + UInt128 = UInt128` or `Float32 * Float64 = Float64`.
+- If one of the operands is signed, the result type will also be signed, otherwise it will be signed. For example, `UInt32 * Int32 = Int64`.
+
+These rules make sure that the result type will be the smallest type which can represent all possible results. While this introduces a risk
+of overflows around the value range boundary, it ensures that calculations are performed quickly using the maximum native integer width of
+64 bit. This behavior also guarantees compatibility with many other databases which provide 64 bit integers (BIGINT) as the biggest integer
+type.
Example:
@@ -22,8 +33,6 @@ SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0
└───────────────┴────────────────────────┴─────────────────────────────────┴──────────────────────────────────────────┘
```
-Arithmetic functions work for any pair of `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64`, `Float32`, or `Float64` values.
-
Overflows are produced the same way as in C++.
## plus
@@ -68,7 +77,7 @@ Alias: `a \* b` (operator)
## divide
-Calculates the quotient of two values `a` and `b`. The result is always a floating-point value. If you need integer division, you can use the `intDiv` function.
+Calculates the quotient of two values `a` and `b`. The result type is always [Float64](../../sql-reference/data-types/float.md). Integer division is provided by the `intDiv` function.
Division by 0 returns `inf`, `-inf`, or `nan`.
@@ -84,7 +93,7 @@ Alias: `a / b` (operator)
Performs an integer division of two values `a` by `b`, i.e. computes the quotient rounded down to the next smallest integer.
-The result has the same type as the dividend (the first parameter).
+The result has the same width as the dividend (the first parameter).
An exception is thrown when dividing by zero, when the quotient does not fit in the range of the dividend, or when dividing a minimal negative number by minus one.
@@ -135,7 +144,7 @@ intDivOrZero(a, b)
Calculates the remainder of the division of two values `a` by `b`.
-The result type is an integer if both inputs are integers. If one of the inputs is a floating-point number, the result is a floating-point number.
+The result type is an integer if both inputs are integers. If one of the inputs is a floating-point number, the result type is [Float64](../../sql-reference/data-types/float.md).
The remainder is computed like in C++. Truncated division is used for negative numbers.
diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md
index bdd1445c990..44d385312d0 100644
--- a/docs/en/sql-reference/functions/array-functions.md
+++ b/docs/en/sql-reference/functions/array-functions.md
@@ -102,6 +102,8 @@ The function also works for strings.
Can be optimized by enabling the [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [size0](../../sql-reference/data-types/array.md#array-size) subcolumn instead of reading and processing the whole array column. The query `SELECT length(arr) FROM table` transforms to `SELECT arr.size0 FROM TABLE`.
+Alias: `OCTET_LENGTH`
+
## emptyArrayUInt8, emptyArrayUInt16, emptyArrayUInt32, emptyArrayUInt64
## emptyArrayInt8, emptyArrayInt16, emptyArrayInt32, emptyArrayInt64
@@ -142,6 +144,7 @@ range([start, ] end [, step])
- All arguments `start`, `end`, `step` must be below data types: `UInt8`, `UInt16`, `UInt32`, `UInt64`,`Int8`, `Int16`, `Int32`, `Int64`, as well as elements of the returned array, which's type is a super type of all arguments.
- An exception is thrown if query results in arrays with a total length of more than number of elements specified by the [function_range_max_elements_in_block](../../operations/settings/settings.md#settings-function_range_max_elements_in_block) setting.
+- Returns Null if any argument has Nullable(Nothing) type. An exception is thrown if any argument has Null value (Nullable(T) type).
**Examples**
@@ -230,13 +233,15 @@ hasAll(set, subset)
**Arguments**
- `set` – Array of any type with a set of elements.
-- `subset` – Array of any type with elements that should be tested to be a subset of `set`.
+- `subset` – Array of any type that shares a common supertype with `set` containing elements that should be tested to be a subset of `set`.
**Return values**
- `1`, if `set` contains all of the elements from `subset`.
- `0`, otherwise.
+Raises an exception `NO_COMMON_TYPE` if the set and subset elements do not share a common supertype.
+
**Peculiar properties**
- An empty array is a subset of any array.
@@ -253,7 +258,7 @@ hasAll(set, subset)
`SELECT hasAll(['a', 'b'], ['a'])` returns 1.
-`SELECT hasAll([1], ['a'])` returns 0.
+`SELECT hasAll([1], ['a'])` raises a `NO_COMMON_TYPE` exception.
`SELECT hasAll([[1, 2], [3, 4]], [[1, 2], [3, 5]])` returns 0.
@@ -268,13 +273,15 @@ hasAny(array1, array2)
**Arguments**
- `array1` – Array of any type with a set of elements.
-- `array2` – Array of any type with a set of elements.
+- `array2` – Array of any type that shares a common supertype with `array1`.
**Return values**
- `1`, if `array1` and `array2` have one similar element at least.
- `0`, otherwise.
+Raises an exception `NO_COMMON_TYPE` if the array1 and array2 elements do not share a common supertype.
+
**Peculiar properties**
- `Null` processed as a value.
@@ -288,7 +295,7 @@ hasAny(array1, array2)
`SELECT hasAny([-128, 1., 512], [1])` returns `1`.
-`SELECT hasAny([[1, 2], [3, 4]], ['a', 'c'])` returns `0`.
+`SELECT hasAny([[1, 2], [3, 4]], ['a', 'c'])` raises a `NO_COMMON_TYPE` exception.
`SELECT hasAll([[1, 2], [3, 4]], [[1, 2], [1, 2]])` returns `1`.
@@ -318,6 +325,8 @@ For Example:
- `1`, if `array1` contains `array2`.
- `0`, otherwise.
+Raises an exception `NO_COMMON_TYPE` if the array1 and array2 elements do not share a common supertype.
+
**Peculiar properties**
- The function will return `1` if `array2` is empty.
@@ -339,6 +348,9 @@ For Example:
`SELECT hasSubstr(['a', 'b' , 'c'], ['a', 'c'])` returns 0.
`SELECT hasSubstr([[1, 2], [3, 4], [5, 6]], [[1, 2], [3, 4]])` returns 1.
+i
+`SELECT hasSubstr([1, 2, NULL, 3, 4], ['a'])` raises a `NO_COMMON_TYPE` exception.
+
## indexOf(arr, x)
@@ -869,7 +881,7 @@ A special function. See the section [“ArrayJoin function”](../../sql-referen
## arrayDifference
-Calculates an array of differences between adjacent array elements. The first element of the result array will be 0, the second `a[1] - a[0]`, the third `a[2] - a[1]`, etc. The type of elements in the result array is determined by the type inference rules for subtraction (e.g. `UInt8` - `UInt8` = `Int16`).
+Calculates an array of differences between adjacent array elements. The first element of the result array will be 0, the second `a[1] - a[0]`, the third `a[2] - a[1]`, etc. The type of elements in the result array is determined by the type inference rules for subtraction (e.g. `UInt8` - `UInt8` = `Int16`).
**Syntax**
@@ -987,6 +999,24 @@ SELECT
└──────────────┴───────────┘
```
+## arrayJaccardIndex
+
+Returns the [Jaccard index](https://en.wikipedia.org/wiki/Jaccard_index) of two arrays.
+
+**Example**
+
+Query:
+``` sql
+SELECT arrayJaccardIndex([1, 2], [2, 3]) AS res
+```
+
+Result:
+``` text
+┌─res────────────────┐
+│ 0.3333333333333333 │
+└────────────────────┘
+```
+
## arrayReduce
Applies an aggregate function to array elements and returns its result. The name of the aggregation function is passed as a string in single quotes `'max'`, `'sum'`. When using parametric aggregate functions, the parameter is indicated after the function name in parentheses `'uniqUpTo(6)'`.
diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md
index 280b41e7a5f..87d84425029 100644
--- a/docs/en/sql-reference/functions/date-time-functions.md
+++ b/docs/en/sql-reference/functions/date-time-functions.md
@@ -139,8 +139,8 @@ makeDateTime32(year, month, day, hour, minute, second[, fraction[, precision[, t
## timeZone
-Returns the timezone of the server.
-If the function is executed in the context of a distributed table, it generates a normal column with values relevant to each shard, otherwise it produces a constant value.
+Returns the timezone of the current session, i.e. the value of setting [session_timezone](../../operations/settings/settings.md#session_timezone).
+If the function is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard, otherwise it produces a constant value.
**Syntax**
@@ -156,6 +156,33 @@ Alias: `timezone`.
Type: [String](../../sql-reference/data-types/string.md).
+**See also**
+
+- [serverTimeZone](#serverTimeZone)
+
+## serverTimeZone
+
+Returns the timezone of the server, i.e. the value of setting [timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone).
+If the function is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard. Otherwise, it produces a constant value.
+
+**Syntax**
+
+``` sql
+serverTimeZone()
+```
+
+Alias: `serverTimezone`.
+
+**Returned value**
+
+- Timezone.
+
+Type: [String](../../sql-reference/data-types/string.md).
+
+**See also**
+
+- [timeZone](#timeZone)
+
## toTimeZone
Converts a date or date with time to the specified time zone. Does not change the internal value (number of unix seconds) of the data, only the value's time zone attribute and the value's string representation changes.
@@ -667,10 +694,14 @@ SELECT toDate('2016-12-27') AS date, toWeek(date) AS week0, toWeek(date,1) AS we
Returns year and week for a date. The year in the result may be different from the year in the date argument for the first and the last week of the year.
-The mode argument works exactly like the mode argument to `toWeek()`. For the single-argument syntax, a mode value of 0 is used.
+The mode argument works like the mode argument to `toWeek()`. For the single-argument syntax, a mode value of 0 is used.
`toISOYear()` is a compatibility function that is equivalent to `intDiv(toYearWeek(date,3),100)`.
+:::warning
+The week number returned by `toYearWeek()` can be different from what the `toWeek()` returns. `toWeek()` always returns week number in the context of the given year, and in case `toWeek()` returns `0`, `toYearWeek()` returns the value corresponding to the last week of previous year. See `prev_yearWeek` in example below.
+:::
+
**Syntax**
``` sql
@@ -680,18 +711,18 @@ toYearWeek(t[, mode[, timezone]])
**Example**
``` sql
-SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(date,1) AS yearWeek1, toYearWeek(date,9) AS yearWeek9;
+SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(date,1) AS yearWeek1, toYearWeek(date,9) AS yearWeek9, toYearWeek(toDate('2022-01-01')) AS prev_yearWeek;
```
``` text
-┌───────date─┬─yearWeek0─┬─yearWeek1─┬─yearWeek9─┐
-│ 2016-12-27 │ 201652 │ 201652 │ 201701 │
-└────────────┴───────────┴───────────┴───────────┘
+┌───────date─┬─yearWeek0─┬─yearWeek1─┬─yearWeek9─┬─prev_yearWeek─┐
+│ 2016-12-27 │ 201652 │ 201652 │ 201701 │ 202152 │
+└────────────┴───────────┴───────────┴───────────┴───────────────┘
```
## age
-Returns the `unit` component of the difference between `startdate` and `enddate`. The difference is calculated using a precision of 1 second.
+Returns the `unit` component of the difference between `startdate` and `enddate`. The difference is calculated using a precision of 1 microsecond.
E.g. the difference between `2021-12-29` and `2022-01-01` is 3 days for `day` unit, 0 months for `month` unit, 0 years for `year` unit.
For an alternative to `age`, see function `date\_diff`.
@@ -707,6 +738,8 @@ age('unit', startdate, enddate, [timezone])
- `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md).
Possible values:
+ - `microsecond` (possible abbreviations: `us`, `u`)
+ - `millisecond` (possible abbreviations: `ms`)
- `second` (possible abbreviations: `ss`, `s`)
- `minute` (possible abbreviations: `mi`, `n`)
- `hour` (possible abbreviations: `hh`, `h`)
@@ -782,6 +815,8 @@ Aliases: `dateDiff`, `DATE_DIFF`, `timestampDiff`, `timestamp_diff`, `TIMESTAMP_
- `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md).
Possible values:
+ - `microsecond` (possible abbreviations: `us`, `u`)
+ - `millisecond` (possible abbreviations: `ms`)
- `second` (possible abbreviations: `ss`, `s`)
- `minute` (possible abbreviations: `mi`, `n`)
- `hour` (possible abbreviations: `hh`, `h`)
@@ -1103,6 +1138,8 @@ Result:
Returns the current date and time at the moment of query analysis. The function is a constant expression.
+Alias: `current_timestamp`.
+
**Syntax**
``` sql
@@ -1233,6 +1270,8 @@ Result:
Accepts zero arguments and returns the current date at one of the moments of query analysis.
The same as ‘toDate(now())’.
+Aliases: `curdate`, `current_date`.
+
## yesterday
Accepts zero arguments and returns yesterday’s date at one of the moments of query analysis.
@@ -1410,7 +1449,7 @@ Using replacement fields, you can define a pattern for the resulting string. “
| %n | new-line character (‘’) | |
| %p | AM or PM designation | PM |
| %Q | Quarter (1-4) | 1 |
-| %r | 12-hour HH:MM AM/PM time, equivalent to %H:%i %p | 10:30 PM |
+| %r | 12-hour HH:MM AM/PM time, equivalent to %h:%i %p | 10:30 PM |
| %R | 24-hour HH:MM time, equivalent to %H:%i | 22:33 |
| %s | second (00-59) | 44 |
| %S | second (00-59) | 44 |
diff --git a/docs/en/sql-reference/functions/distance-functions.md b/docs/en/sql-reference/functions/distance-functions.md
index 67affb88a53..1774c22014d 100644
--- a/docs/en/sql-reference/functions/distance-functions.md
+++ b/docs/en/sql-reference/functions/distance-functions.md
@@ -237,6 +237,43 @@ Result:
└────────────────────────────┘
```
+## L2SquaredDistance
+
+Calculates the sum of the squares of the difference between the corresponding elements of two vectors.
+
+**Syntax**
+
+```sql
+L2SquaredDistance(vector1, vector2)
+```
+
+Alias: `distanceL2Squared`.
+
+**Arguments**
+
+- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+- `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L2SquaredDistance([1, 2, 3], [0, 0, 0])
+```
+
+Result:
+
+```response
+┌─L2SquaredDistance([1, 2, 3], [0, 0, 0])─┐
+│ 14 │
+└─────────────────────────────────────────┘
+```
+
## LinfDistance
Calculates the distance between two points (the values of the vectors are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
diff --git a/docs/en/sql-reference/functions/functions-for-nulls.md b/docs/en/sql-reference/functions/functions-for-nulls.md
index 6f82fedaab7..d57b799e94c 100644
--- a/docs/en/sql-reference/functions/functions-for-nulls.md
+++ b/docs/en/sql-reference/functions/functions-for-nulls.md
@@ -8,7 +8,7 @@ sidebar_label: Nullable
## isNull
-Returns whether the argument is [NULL](../../sql-reference/syntax.md#null-literal).
+Returns whether the argument is [NULL](../../sql-reference/syntax.md#null).
``` sql
isNull(x)
diff --git a/docs/en/sql-reference/functions/ip-address-functions.md b/docs/en/sql-reference/functions/ip-address-functions.md
index 0dc1db1161b..33c788a632e 100644
--- a/docs/en/sql-reference/functions/ip-address-functions.md
+++ b/docs/en/sql-reference/functions/ip-address-functions.md
@@ -248,7 +248,7 @@ SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32);
## toIPv4(string)
-An alias to `IPv4StringToNum()` that takes a string form of IPv4 address and returns value of [IPv4](../../sql-reference/data-types/domains/ipv4.md) type, which is binary equal to value returned by `IPv4StringToNum()`.
+An alias to `IPv4StringToNum()` that takes a string form of IPv4 address and returns value of [IPv4](../../sql-reference/data-types/ipv4.md) type, which is binary equal to value returned by `IPv4StringToNum()`.
``` sql
WITH
@@ -296,7 +296,7 @@ Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns null
## toIPv6
-Converts a string form of IPv6 address to [IPv6](../../sql-reference/data-types/domains/ipv6.md) type. If the IPv6 address has an invalid format, returns an empty value.
+Converts a string form of IPv6 address to [IPv6](../../sql-reference/data-types/ipv6.md) type. If the IPv6 address has an invalid format, returns an empty value.
Similar to [IPv6StringToNum](#ipv6stringtonums) function, which converts IPv6 address to binary format.
If the input string contains a valid IPv4 address, then the IPv6 equivalent of the IPv4 address is returned.
@@ -315,7 +315,7 @@ toIPv6(string)
- IP address.
-Type: [IPv6](../../sql-reference/data-types/domains/ipv6.md).
+Type: [IPv6](../../sql-reference/data-types/ipv6.md).
**Examples**
diff --git a/docs/en/sql-reference/functions/nlp-functions.md b/docs/en/sql-reference/functions/nlp-functions.md
index f10415783a5..bb127a939f3 100644
--- a/docs/en/sql-reference/functions/nlp-functions.md
+++ b/docs/en/sql-reference/functions/nlp-functions.md
@@ -4,6 +4,8 @@ sidebar_position: 130
sidebar_label: NLP (experimental)
---
+# Natural Language Processing (NLP) Functions
+
:::note
This is an experimental feature that is currently in development and is not ready for general use. It will change in unpredictable backwards-incompatible ways in future releases. Set `allow_experimental_nlp_functions = 1` to enable it.
:::
diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md
index 5175bbf0615..527ce2434c0 100644
--- a/docs/en/sql-reference/functions/string-functions.md
+++ b/docs/en/sql-reference/functions/string-functions.md
@@ -90,6 +90,8 @@ Returns the length of a string in bytes (not: in characters or Unicode code poin
The function also works for arrays.
+Alias: `OCTET_LENGTH`
+
## lengthUTF8
Returns the length of a string in Unicode code points (not: in bytes or characters). It assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
@@ -573,6 +575,42 @@ Alias:
Like `substring` but for Unicode code points. Assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
+
+## substringIndex(s, delim, count)
+
+Returns the substring of `s` before `count` occurrences of the delimiter `delim`, as in Spark or MySQL.
+
+**Syntax**
+
+```sql
+substringIndex(s, delim, count)
+```
+Alias: `SUBSTRING_INDEX`
+
+
+**Arguments**
+
+- s: The string to extract substring from. [String](../../sql-reference/data-types/string.md).
+- delim: The character to split. [String](../../sql-reference/data-types/string.md).
+- count: The number of occurrences of the delimiter to count before extracting the substring. If count is positive, everything to the left of the final delimiter (counting from the left) is returned. If count is negative, everything to the right of the final delimiter (counting from the right) is returned. [UInt or Int](../data-types/int-uint.md)
+
+**Example**
+
+``` sql
+SELECT substringIndex('www.clickhouse.com', '.', 2)
+```
+
+Result:
+```
+┌─substringIndex('www.clickhouse.com', '.', 2)─┐
+│ www.clickhouse │
+└──────────────────────────────────────────────┘
+```
+
+## substringIndexUTF8(s, delim, count)
+
+Like `substringIndex` but for Unicode code points. Assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
+
## appendTrailingCharIfAbsent
Appends character `c` to string `s` if `s` is non-empty and does not end with character `c`.
@@ -1253,3 +1291,48 @@ Result:
│ A240 │
└──────────────────┘
```
+
+## initcap
+
+Convert the first letter of each word to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters.
+
+## initcapUTF8
+
+Like [initcap](#initcap), assuming that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
+
+Does not detect the language, e.g. for Turkish the result might not be exactly correct (i/İ vs. i/I).
+
+If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point.
+
+## firstLine
+
+Returns the first line from a multi-line string.
+
+**Syntax**
+
+```sql
+firstLine(val)
+```
+
+**Arguments**
+
+- `val` - Input value. [String](../data-types/string.md)
+
+**Returned value**
+
+- The first line of the input value or the whole value if there is no line
+ separators. [String](../data-types/string.md)
+
+**Example**
+
+```sql
+select firstLine('foo\nbar\nbaz');
+```
+
+Result:
+
+```result
+┌─firstLine('foo\nbar\nbaz')─┐
+│ foo │
+└────────────────────────────┘
+```
diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md
index 3d8f89f7295..c10a1036677 100644
--- a/docs/en/sql-reference/functions/string-search-functions.md
+++ b/docs/en/sql-reference/functions/string-search-functions.md
@@ -631,3 +631,53 @@ Result:
│ 100 │ 200 │ 100-200 │ 100 │
└──────────────────────────────────────────────┴──────────────────────────────────────────────┴──────────────────────────────────────────────┴───────────────────────────────────────────┘
```
+
+## hasSubsequence
+
+Returns 1 if needle is a subsequence of haystack, or 0 otherwise.
+A subsequence of a string is a sequence that can be derived from the given string by deleting zero or more elements without changing the order of the remaining elements.
+
+
+**Syntax**
+
+``` sql
+hasSubsequence(haystack, needle)
+```
+
+**Arguments**
+
+- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Subsequence to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
+
+**Returned values**
+
+- 1, if needle is a subsequence of haystack.
+- 0, otherwise.
+
+Type: `UInt8`.
+
+**Examples**
+
+``` sql
+SELECT hasSubsequence('garbage', 'arg') ;
+```
+
+Result:
+
+``` text
+┌─hasSubsequence('garbage', 'arg')─┐
+│ 1 │
+└──────────────────────────────────┘
+```
+
+## hasSubsequenceCaseInsensitive
+
+Like [hasSubsequence](#hasSubsequence) but searches case-insensitively.
+
+## hasSubsequenceUTF8
+
+Like [hasSubsequence](#hasSubsequence) but assumes `haystack` and `needle` are UTF-8 encoded strings.
+
+## hasSubsequenceCaseInsensitiveUTF8
+
+Like [hasSubsequenceUTF8](#hasSubsequenceUTF8) but searches case-insensitively.
\ No newline at end of file
diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md
index 1739920c9f0..7ed2deaeda6 100644
--- a/docs/en/sql-reference/functions/tuple-functions.md
+++ b/docs/en/sql-reference/functions/tuple-functions.md
@@ -22,14 +22,15 @@ tuple(x, y, …)
A function that allows getting a column from a tuple.
-If the second argument is a number `n`, it is the column index, starting from 1. If the second argument is a string `s`, it represents the name of the element. Besides, we can provide the third optional argument, such that when index out of bounds or element for such name does not exist, the default value returned instead of throw exception. The second and third arguments if provided are always must be constant. There is no cost to execute the function.
+If the second argument is a number `index`, it is the column index, starting from 1. If the second argument is a string `name`, it represents the name of the element. Besides, we can provide the third optional argument, such that when index out of bounds or no element exist for the name, the default value returned instead of throwing an exception. The second and third arguments, if provided, must be constants. There is no cost to execute the function.
-The function implements the operator `x.n` and `x.s`.
+The function implements operators `x.index` and `x.name`.
**Syntax**
``` sql
-tupleElement(tuple, n/s [, default_value])
+tupleElement(tuple, index, [, default_value])
+tupleElement(tuple, name, [, default_value])
```
## untuple
diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 214c885bc0e..36f40b37238 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -399,7 +399,11 @@ toDateTime(expr[, time_zone ])
- `expr` — The value. [String](/docs/en/sql-reference/data-types/string.md), [Int](/docs/en/sql-reference/data-types/int-uint.md), [Date](/docs/en/sql-reference/data-types/date.md) or [DateTime](/docs/en/sql-reference/data-types/datetime.md).
- `time_zone` — Time zone. [String](/docs/en/sql-reference/data-types/string.md).
-If `expr` is a number, it is interpreted as the number of seconds since the beginning of the Unix Epoch (as Unix timestamp).
+:::note
+If `expr` is a number, it is interpreted as the number of seconds since the beginning of the Unix Epoch (as Unix timestamp).
+If `expr` is a [String](/docs/en/sql-reference/data-types/string.md), it may be interpreted as a Unix timestamp or as a string representation of date / date with time.
+Thus, parsing of short numbers' string representations (up to 4 digits) is explicitly disabled due to ambiguity, e.g. a string `'1999'` may be both a year (an incomplete string representation of Date / DateTime) or a unix timestamp. Longer numeric strings are allowed.
+:::
**Returned value**
diff --git a/docs/en/sql-reference/functions/udf.md b/docs/en/sql-reference/functions/udf.md
index 9c6b1b0c66b..51734beed03 100644
--- a/docs/en/sql-reference/functions/udf.md
+++ b/docs/en/sql-reference/functions/udf.md
@@ -171,12 +171,13 @@ Result:
└──────────────────────────────┘
```
-Executable user defined functions can take constant parameters configured in `command` setting (works only for user defined functions with `executable` type).
+Executable user defined functions can take constant parameters configured in `command` setting (works only for user defined functions with `executable` type). It also requires the `execute_direct` option (to ensure no shell argument expansion vulnerability).
File `test_function_parameter_python.xml` (`/etc/clickhouse-server/test_function_parameter_python.xml` with default path settings).
```xml
executable
+ true
test_function_parameter_python
String
diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md
index 378f41c1199..6ceb9b5849e 100644
--- a/docs/en/sql-reference/statements/alter/column.md
+++ b/docs/en/sql-reference/statements/alter/column.md
@@ -213,7 +213,7 @@ Removes one of the column properties: `DEFAULT`, `ALIAS`, `MATERIALIZED`, `CODEC
Syntax:
```sql
-ALTER TABLE table_name MODIFY column_name REMOVE property;
+ALTER TABLE table_name MODIFY COLUMN column_name REMOVE property;
```
**Example**
@@ -232,6 +232,7 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL;
Materializes or updates a column with an expression for a default value (`DEFAULT` or `MATERIALIZED`).
It is used if it is necessary to add or update a column with a complicated expression, because evaluating such an expression directly on `SELECT` executing turns out to be expensive.
+Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
Syntax:
diff --git a/docs/en/sql-reference/statements/alter/index.md b/docs/en/sql-reference/statements/alter/index.md
index 7a687a067aa..7dadc2be5b2 100644
--- a/docs/en/sql-reference/statements/alter/index.md
+++ b/docs/en/sql-reference/statements/alter/index.md
@@ -60,7 +60,7 @@ You can specify how long (in seconds) to wait for inactive replicas to execute a
For all `ALTER` queries, if `alter_sync = 2` and some replicas are not active for more than the time, specified in the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown.
:::
-For `ALTER TABLE ... UPDATE|DELETE` queries the synchronicity is defined by the [mutations_sync](/docs/en/operations/settings/settings.md/#mutations_sync) setting.
+For `ALTER TABLE ... UPDATE|DELETE|MATERIALIZE INDEX|MATERIALIZE PROJECTION|MATERIALIZE COLUMN` queries the synchronicity is defined by the [mutations_sync](/docs/en/operations/settings/settings.md/#mutations_sync) setting.
## Related content
diff --git a/docs/en/sql-reference/statements/alter/projection.md b/docs/en/sql-reference/statements/alter/projection.md
index b7399442d41..fb438927089 100644
--- a/docs/en/sql-reference/statements/alter/projection.md
+++ b/docs/en/sql-reference/statements/alter/projection.md
@@ -142,19 +142,19 @@ The following operations with [projections](/docs/en/engines/table-engines/merge
## ADD PROJECTION
-`ALTER TABLE [db].name ADD PROJECTION [IF NOT EXISTS] name ( SELECT [GROUP BY] [ORDER BY] )` - Adds projection description to tables metadata.
+`ALTER TABLE [db.]name [ON CLUSTER cluster] ADD PROJECTION [IF NOT EXISTS] name ( SELECT [GROUP BY] [ORDER BY] )` - Adds projection description to tables metadata.
## DROP PROJECTION
-`ALTER TABLE [db].name DROP PROJECTION [IF EXISTS] name` - Removes projection description from tables metadata and deletes projection files from disk. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
+`ALTER TABLE [db.]name [ON CLUSTER cluster] DROP PROJECTION [IF EXISTS] name` - Removes projection description from tables metadata and deletes projection files from disk. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
## MATERIALIZE PROJECTION
-`ALTER TABLE [db.]table MATERIALIZE PROJECTION name IN PARTITION partition_name` - The query rebuilds the projection `name` in the partition `partition_name`. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
+`ALTER TABLE [db.]table [ON CLUSTER cluster] MATERIALIZE PROJECTION [IF EXISTS] name [IN PARTITION partition_name]` - The query rebuilds the projection `name` in the partition `partition_name`. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
## CLEAR PROJECTION
-`ALTER TABLE [db.]table CLEAR PROJECTION [IF EXISTS] name IN PARTITION partition_name` - Deletes projection files from disk without removing description. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
+`ALTER TABLE [db.]table [ON CLUSTER cluster] CLEAR PROJECTION [IF EXISTS] name [IN PARTITION partition_name]` - Deletes projection files from disk without removing description. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
The commands `ADD`, `DROP` and `CLEAR` are lightweight in a sense that they only change metadata or remove files.
diff --git a/docs/en/sql-reference/statements/alter/sample-by.md b/docs/en/sql-reference/statements/alter/sample-by.md
index b20f3c7b5d3..ccad792f853 100644
--- a/docs/en/sql-reference/statements/alter/sample-by.md
+++ b/docs/en/sql-reference/statements/alter/sample-by.md
@@ -5,15 +5,28 @@ sidebar_label: SAMPLE BY
title: "Manipulating Sampling-Key Expressions"
---
-Syntax:
+# Manipulating SAMPLE BY expression
+
+The following operations are available:
+
+## MODIFY
``` sql
ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY SAMPLE BY new_expression
```
-The command changes the [sampling key](../../../engines/table-engines/mergetree-family/mergetree.md) of the table to `new_expression` (an expression or a tuple of expressions).
+The command changes the [sampling key](../../../engines/table-engines/mergetree-family/mergetree.md) of the table to `new_expression` (an expression or a tuple of expressions). The primary key must contain the new sample key.
-The command is lightweight in the sense that it only changes metadata. The primary key must contain the new sample key.
+## REMOVE
+
+``` sql
+ALTER TABLE [db].name [ON CLUSTER cluster] REMOVE SAMPLE BY
+```
+
+The command removes the [sampling key](../../../engines/table-engines/mergetree-family/mergetree.md) of the table.
+
+
+The commands `MODIFY` and `REMOVE` are lightweight in the sense that they only change metadata or remove files.
:::note
It only works for tables in the [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) family (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) tables).
diff --git a/docs/en/sql-reference/statements/alter/skipping-index.md b/docs/en/sql-reference/statements/alter/skipping-index.md
index 4194731d33a..42fd12d9487 100644
--- a/docs/en/sql-reference/statements/alter/skipping-index.md
+++ b/docs/en/sql-reference/statements/alter/skipping-index.md
@@ -10,15 +10,25 @@ sidebar_label: INDEX
The following operations are available:
-- `ALTER TABLE [db].table_name [ON CLUSTER cluster] ADD INDEX name expression TYPE type [GRANULARITY value] [FIRST|AFTER name]` - Adds index description to tables metadata.
+## ADD INDEX
-- `ALTER TABLE [db].table_name [ON CLUSTER cluster] DROP INDEX name` - Removes index description from tables metadata and deletes index files from disk. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
+`ALTER TABLE [db.]table_name [ON CLUSTER cluster] ADD INDEX [IF NOT EXISTS] name expression TYPE type [GRANULARITY value] [FIRST|AFTER name]` - Adds index description to tables metadata.
-- `ALTER TABLE [db.]table_name [ON CLUSTER cluster] MATERIALIZE INDEX name [IN PARTITION partition_name]` - Rebuilds the secondary index `name` for the specified `partition_name`. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations). If `IN PARTITION` part is omitted then it rebuilds the index for the whole table data.
+## DROP INDEX
-The first two commands are lightweight in a sense that they only change metadata or remove files.
+`ALTER TABLE [db.]table_name [ON CLUSTER cluster] DROP INDEX [IF EXISTS] name` - Removes index description from tables metadata and deletes index files from disk. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
-Also, they are replicated, syncing indices metadata via ZooKeeper.
+## MATERIALIZE INDEX
+
+`ALTER TABLE [db.]table_name [ON CLUSTER cluster] MATERIALIZE INDEX [IF EXISTS] name [IN PARTITION partition_name]` - Rebuilds the secondary index `name` for the specified `partition_name`. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations). If `IN PARTITION` part is omitted then it rebuilds the index for the whole table data.
+
+## CLEAR INDEX
+
+`ALTER TABLE [db.]table_name [ON CLUSTER cluster] CLEAR INDEX [IF EXISTS] name [IN PARTITION partition_name]` - Deletes the secondary index files from disk without removing description. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
+
+
+The commands `ADD`, `DROP`, and `CLEAR` are lightweight in the sense that they only change metadata or remove files.
+Also, they are replicated, syncing indices metadata via ClickHouse Keeper or ZooKeeper.
:::note
Index manipulation is supported only for tables with [`*MergeTree`](/docs/en/engines/table-engines/mergetree-family/mergetree.md) engine (including [replicated](/docs/en/engines/table-engines/mergetree-family/replication.md) variants).
diff --git a/docs/en/sql-reference/statements/create/dictionary.md b/docs/en/sql-reference/statements/create/dictionary.md
index 29c72d62f24..c0a153c5660 100644
--- a/docs/en/sql-reference/statements/create/dictionary.md
+++ b/docs/en/sql-reference/statements/create/dictionary.md
@@ -82,6 +82,35 @@ LIFETIME(MIN 0 MAX 1000)
LAYOUT(FLAT())
```
+:::note
+When using the SQL console in [ClickHouse Cloud](https://clickhouse.com), you must specify a user (`default` or any other user with the role `default_role`) and password when creating a dictionary.
+:::note
+
+```sql
+CREATE USER IF NOT EXISTS clickhouse_admin
+IDENTIFIED WITH sha256_password BY 'passworD43$x';
+
+GRANT default_role TO clickhouse_admin;
+
+CREATE DATABASE foo_db;
+
+CREATE TABLE foo_db.source_table (
+ id UInt64,
+ value String
+) ENGINE = MergeTree
+PRIMARY KEY id;
+
+CREATE DICTIONARY foo_db.id_value_dictionary
+(
+ id UInt64,
+ value String
+)
+PRIMARY KEY id
+SOURCE(CLICKHOUSE(TABLE 'source_table' USER 'clickhouse_admin' PASSWORD 'passworD43$x' DB 'foo_db' ))
+LAYOUT(FLAT())
+LIFETIME(MIN 0 MAX 1000);
+```
+
### Create a dictionary from a table in a remote ClickHouse service
Input table (in the remote ClickHouse service) `source_table`:
diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md
index de44a001472..1a72f89fb1f 100644
--- a/docs/en/sql-reference/statements/create/table.md
+++ b/docs/en/sql-reference/statements/create/table.md
@@ -380,11 +380,15 @@ High compression levels are useful for asymmetric scenarios, like compress once,
`DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply:
-- DEFLATE_QPL is experimental and can only be used after setting configuration parameter `allow_experimental_codecs=1`.
+- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_deflate_qpl_codec = 1`.
- DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details.
- DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details.
- DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled.
+:::note
+DEFLATE_QPL is not available in ClickHouse Cloud.
+:::
+
### Specialized Codecs
These codecs are designed to make compression more effective by using specific features of data. Some of these codecs do not compress data themself. Instead, they prepare the data for a common purpose codec, which compresses it better than without this preparation.
diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md
index 10b15638152..11026340a0f 100644
--- a/docs/en/sql-reference/statements/create/view.md
+++ b/docs/en/sql-reference/statements/create/view.md
@@ -97,7 +97,7 @@ This is an experimental feature that may change in backwards-incompatible ways i
:::
```sql
-CREATE LIVE VIEW [IF NOT EXISTS] [db.]table_name [WITH [TIMEOUT [value_in_sec] [AND]] [REFRESH [value_in_sec]]] AS SELECT ...
+CREATE LIVE VIEW [IF NOT EXISTS] [db.]table_name [WITH REFRESH [value_in_sec]] AS SELECT ...
```
Live views store result of the corresponding [SELECT](../../../sql-reference/statements/select/index.md) query and are updated any time the result of the query changes. Query result as well as partial result needed to combine with new data are stored in memory providing increased performance for repeated queries. Live views can provide push notifications when query result changes using the [WATCH](../../../sql-reference/statements/watch.md) query.
diff --git a/docs/en/sql-reference/statements/select/join.md b/docs/en/sql-reference/statements/select/join.md
index 08ffae838f8..7971b3ba275 100644
--- a/docs/en/sql-reference/statements/select/join.md
+++ b/docs/en/sql-reference/statements/select/join.md
@@ -21,6 +21,9 @@ Expressions from `ON` clause and columns from `USING` clause are called “join
## Related Content
- Blog: [ClickHouse: A Blazingly Fast DBMS with Full SQL Join Support - Part 1](https://clickhouse.com/blog/clickhouse-fully-supports-joins)
+- Blog: [ClickHouse: A Blazingly Fast DBMS with Full SQL Join Support - Under the Hood - Part 2](https://clickhouse.com/blog/clickhouse-fully-supports-joins-hash-joins-part2)
+- Blog: [ClickHouse: A Blazingly Fast DBMS with Full SQL Join Support - Under the Hood - Part 3](https://clickhouse.com/blog/clickhouse-fully-supports-joins-full-sort-partial-merge-part3)
+- Blog: [ClickHouse: A Blazingly Fast DBMS with Full SQL Join Support - Under the Hood - Part 4](https://clickhouse.com/blog/clickhouse-fully-supports-joins-direct-join-part4)
## Supported Types of JOIN
diff --git a/docs/en/sql-reference/statements/select/with.md b/docs/en/sql-reference/statements/select/with.md
index 4654f249548..a59ef463419 100644
--- a/docs/en/sql-reference/statements/select/with.md
+++ b/docs/en/sql-reference/statements/select/with.md
@@ -5,7 +5,27 @@ sidebar_label: WITH
# WITH Clause
-ClickHouse supports Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), that is provides to use results of `WITH` clause in the rest of `SELECT` query. Named subqueries can be included to the current and child query context in places where table objects are allowed. Recursion is prevented by hiding the current level CTEs from the WITH expression.
+ClickHouse supports Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)) and substitutes the code defined in the `WITH` clause in all places of use for the rest of `SELECT` query. Named subqueries can be included to the current and child query context in places where table objects are allowed. Recursion is prevented by hiding the current level CTEs from the WITH expression.
+
+Please note that CTEs do not guarantee the same results in all places they are called because the query will be re-executed for each use case.
+
+An example of such behavior is below
+``` sql
+with cte_numbers as
+(
+ select
+ num
+ from generateRandom('num UInt64', NULL)
+ limit 1000000
+)
+select
+ count()
+from cte_numbers
+where num in (select num from cte_numbers)
+```
+If CTEs were to pass exactly the results and not just a piece of code, you would always see `1000000`
+
+However, due to the fact that we are referring `cte_numbers` twice, random numbers are generated each time and, accordingly, we see different random results, `280501, 392454, 261636, 196227` and so on...
## Syntax
diff --git a/docs/en/sql-reference/statements/set.md b/docs/en/sql-reference/statements/set.md
index 14f523adc3b..3e5e86eccf7 100644
--- a/docs/en/sql-reference/statements/set.md
+++ b/docs/en/sql-reference/statements/set.md
@@ -10,7 +10,7 @@ sidebar_label: SET
SET param = value
```
-Assigns `value` to the `param` [setting](../../operations/settings/index.md) for the current session. You cannot change [server settings](../../operations/server-configuration-parameters/index.md) this way.
+Assigns `value` to the `param` [setting](../../operations/settings/index.md) for the current session. You cannot change [server settings](../../operations/server-configuration-parameters/settings.md) this way.
You can also set all the values from the specified settings profile in a single query.
diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md
index f96eb55aa45..1c399d2072b 100644
--- a/docs/en/sql-reference/statements/show.md
+++ b/docs/en/sql-reference/statements/show.md
@@ -205,7 +205,7 @@ The optional keyword `EXTENDED` currently has no effect, it only exists for MySQ
The optional keyword `FULL` causes the output to include the collation, comment and privilege columns.
-`SHOW COLUMNS` produces a result table with the following structure:
+The statement produces a result table with the following structure:
- field - The name of the column (String)
- type - The column data type (String)
- null - If the column data type is Nullable (UInt8)
@@ -272,6 +272,10 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2
Displays a list of primary and data skipping indexes of a table.
+This statement mostly exists for compatibility with MySQL. System tables [system.tables](../../operations/system-tables/tables.md) (for
+primary keys) and [system.data_skipping_indices](../../operations/system-tables/data_skipping_indices.md) (for data skipping indices)
+provide equivalent information but in a fashion more native to ClickHouse.
+
```sql
SHOW [EXTENDED] {INDEX | INDEXES | INDICES | KEYS } {FROM | IN} [{FROM | IN} ] [WHERE ] [INTO OUTFILE ] [FORMAT ]
```
@@ -281,22 +285,22 @@ equivalent. If no database is specified, the query assumes the current database
The optional keyword `EXTENDED` currently has no effect, it only exists for MySQL compatibility.
-`SHOW INDEX` produces a result table with the following structure:
-- table - The name of the table (String)
-- non_unique - 0 if the index can contain duplicates, 1 otherwise (UInt8)
-- key_name - The name of the index, `PRIMARY` if the index is a primary key index (String)
-- seq_in_index - Currently unused
-- column_name - Currently unused
-- collation - The sorting of the column in the index, `A` if ascending, `D` if descending, `NULL` if unsorted (Nullable(String))
-- cardinality - Currently unused
-- sub_part - Currently unused
-- packed - Currently unused
+The statement produces a result table with the following structure:
+- table - The name of the table. (String)
+- non_unique - Always `1` as ClickHouse does not support uniqueness constraints. (UInt8)
+- key_name - The name of the index, `PRIMARY` if the index is a primary key index. (String)
+- seq_in_index - For a primary key index, the position of the column starting from `1`. For a data skipping index: always `1`. (UInt8)
+- column_name - For a primary key index, the name of the column. For a data skipping index: `''` (empty string), see field "expression". (String)
+- collation - The sorting of the column in the index: `A` if ascending, `D` if descending, `NULL` if unsorted. (Nullable(String))
+- cardinality - An estimation of the index cardinality (number of unique values in the index). Currently always 0. (UInt64)
+- sub_part - Always `NULL` because ClickHouse does not support index prefixes like MySQL. (Nullable(String))
+- packed - Always `NULL` because ClickHouse does not support packed indexes (like MySQL). (Nullable(String))
- null - Currently unused
-- index_type - The index type, e.g. `primary`, `minmax`, `bloom_filter` etc. (String)
-- comment - Currently unused
-- index_comment - Currently unused
-- visible - If the index is visible to the optimizer, always `YES` (String)
-- expression - The index expression (String)
+- index_type - The index type, e.g. `PRIMARY`, `MINMAX`, `BLOOM_FILTER` etc. (String)
+- comment - Additional information about the index, currently always `''` (empty string). (String)
+- index_comment - `''` (empty string) because indexes in ClickHouse cannot have a `COMMENT` field (like in MySQL). (String)
+- visible - If the index is visible to the optimizer, always `YES`. (String)
+- expression - For a data skipping index, the index expression. For a primary key index: `''` (empty string). (String)
**Examples**
@@ -310,11 +314,12 @@ Result:
``` text
┌─table─┬─non_unique─┬─key_name─┬─seq_in_index─┬─column_name─┬─collation─┬─cardinality─┬─sub_part─┬─packed─┬─null─┬─index_type───┬─comment─┬─index_comment─┬─visible─┬─expression─┐
-│ tbl │ 0 │ blf_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ bloom_filter │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ d, b │
-│ tbl │ 0 │ mm1_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ minmax │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ a, c, d │
-│ tbl │ 0 │ mm2_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ minmax │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ c, d, e │
-│ tbl │ 0 │ PRIMARY │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ A │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ primary │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ c, a │
-│ tbl │ 0 │ set_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ set │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ e │
+│ tbl │ 1 │ blf_idx │ 1 │ 1 │ ᴺᵁᴸᴸ │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ BLOOM_FILTER │ │ │ YES │ d, b │
+│ tbl │ 1 │ mm1_idx │ 1 │ 1 │ ᴺᵁᴸᴸ │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ MINMAX │ │ │ YES │ a, c, d │
+│ tbl │ 1 │ mm2_idx │ 1 │ 1 │ ᴺᵁᴸᴸ │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ MINMAX │ │ │ YES │ c, d, e │
+│ tbl │ 1 │ PRIMARY │ 1 │ c │ A │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ PRIMARY │ │ │ YES │ │
+│ tbl │ 1 │ PRIMARY │ 2 │ a │ A │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ PRIMARY │ │ │ YES │ │
+│ tbl │ 1 │ set_idx │ 1 │ 1 │ ᴺᵁᴸᴸ │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ SET │ │ │ YES │ e │
└───────┴────────────┴──────────┴──────────────┴─────────────┴───────────┴─────────────┴──────────┴────────┴──────┴──────────────┴─────────┴───────────────┴─────────┴────────────┘
```
diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md
index 65a35f03fbe..fb601cd5d35 100644
--- a/docs/en/sql-reference/statements/system.md
+++ b/docs/en/sql-reference/statements/system.md
@@ -414,3 +414,29 @@ Will do sync syscall.
```sql
SYSTEM SYNC FILE CACHE [ON CLUSTER cluster_name]
```
+
+
+### SYSTEM STOP LISTEN
+
+Closes the socket and gracefully terminates the existing connections to the server on the specified port with the specified protocol.
+
+However, if the corresponding protocol settings were not specified in the clickhouse-server configuration, this command will have no effect.
+
+```sql
+SYSTEM STOP LISTEN [ON CLUSTER cluster_name] [QUERIES ALL | QUERIES DEFAULT | QUERIES CUSTOM | TCP | TCP_WITH_PROXY | TCP_SECURE | HTTP | HTTPS | MYSQL | GRPC | POSTGRESQL | PROMETHEUS | CUSTOM 'protocol']
+```
+
+- If `CUSTOM 'protocol'` modifier is specified, the custom protocol with the specified name defined in the protocols section of the server configuration will be stopped.
+- If `QUERIES ALL` modifier is specified, all protocols are stopped.
+- If `QUERIES DEFAULT` modifier is specified, all default protocols are stopped.
+- If `QUERIES CUSTOM` modifier is specified, all custom protocols are stopped.
+
+### SYSTEM START LISTEN
+
+Allows new connections to be established on the specified protocols.
+
+However, if the server on the specified port and protocol was not stopped using the SYSTEM STOP LISTEN command, this command will have no effect.
+
+```sql
+SYSTEM START LISTEN [ON CLUSTER cluster_name] [QUERIES ALL | QUERIES DEFAULT | QUERIES CUSTOM | TCP | TCP_WITH_PROXY | TCP_SECURE | HTTP | HTTPS | MYSQL | GRPC | POSTGRESQL | PROMETHEUS | CUSTOM 'protocol']
+```
diff --git a/docs/en/sql-reference/table-functions/azureBlobStorage.md b/docs/en/sql-reference/table-functions/azureBlobStorage.md
index 5175aabd5d1..7bb5d892c47 100644
--- a/docs/en/sql-reference/table-functions/azureBlobStorage.md
+++ b/docs/en/sql-reference/table-functions/azureBlobStorage.md
@@ -1,5 +1,6 @@
---
slug: /en/sql-reference/table-functions/azureBlobStorage
+sidebar_position: 10
sidebar_label: azureBlobStorage
keywords: [azure blob storage]
---
@@ -34,16 +35,16 @@ A table with the specified structure for reading or writing data in the specifie
Write data into azure blob storage using the following :
```sql
-INSERT INTO TABLE FUNCTION azureBlobStorage('http://azurite1:10000/devstoreaccount1',
+INSERT INTO TABLE FUNCTION azureBlobStorage('http://azurite1:10000/devstoreaccount1',
'test_container', 'test_{_partition_id}.csv', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==',
'CSV', 'auto', 'column1 UInt32, column2 UInt32, column3 UInt32') PARTITION BY column3 VALUES (1, 2, 3), (3, 2, 1), (78, 43, 3);
```
-And then it can be read using
+And then it can be read using
```sql
-SELECT * FROM azureBlobStorage('http://azurite1:10000/devstoreaccount1',
- 'test_container', 'test_1.csv', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==',
+SELECT * FROM azureBlobStorage('http://azurite1:10000/devstoreaccount1',
+ 'test_container', 'test_1.csv', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==',
'CSV', 'auto', 'column1 UInt32, column2 UInt32, column3 UInt32');
```
diff --git a/docs/en/sql-reference/table-functions/cluster.md b/docs/en/sql-reference/table-functions/cluster.md
index cff8402a200..a083c6b89a6 100644
--- a/docs/en/sql-reference/table-functions/cluster.md
+++ b/docs/en/sql-reference/table-functions/cluster.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/cluster
-sidebar_position: 50
+sidebar_position: 30
sidebar_label: cluster
title: "cluster, clusterAllReplicas"
---
@@ -9,7 +9,7 @@ Allows to access all shards in an existing cluster which configured in `remote_s
`clusterAllReplicas` function — same as `cluster`, but all replicas are queried. Each replica in a cluster is used as a separate shard/connection.
-:::note
+:::note
All available clusters are listed in the [system.clusters](../../operations/system-tables/clusters.md) table.
:::
@@ -23,9 +23,9 @@ clusterAllReplicas(['cluster_name', db, table, sharding_key])
```
**Arguments**
-- `cluster_name` – Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers, set `default` if not specify.
-- `db.table` or `db`, `table` - Name of a database and a table.
-- `sharding_key` - A sharding key. Optional. Needs to be specified if the cluster has more than one shard.
+- `cluster_name` – Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers, set `default` if not specified.
+- `db.table` or `db`, `table` - Name of a database and a table.
+- `sharding_key` - A sharding key. Optional. Needs to be specified if the cluster has more than one shard.
**Returned value**
diff --git a/docs/en/sql-reference/table-functions/deltalake.md b/docs/en/sql-reference/table-functions/deltalake.md
index f1cc4659a2a..885d8df6a1e 100644
--- a/docs/en/sql-reference/table-functions/deltalake.md
+++ b/docs/en/sql-reference/table-functions/deltalake.md
@@ -1,6 +1,7 @@
---
slug: /en/sql-reference/table-functions/deltalake
-sidebar_label: DeltaLake
+sidebar_position: 45
+sidebar_label: deltaLake
---
# deltaLake Table Function
diff --git a/docs/en/sql-reference/table-functions/dictionary.md b/docs/en/sql-reference/table-functions/dictionary.md
index 73d5039a64b..d34bc86e0cd 100644
--- a/docs/en/sql-reference/table-functions/dictionary.md
+++ b/docs/en/sql-reference/table-functions/dictionary.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/dictionary
-sidebar_position: 54
+sidebar_position: 47
sidebar_label: dictionary
title: dictionary
---
diff --git a/docs/en/sql-reference/table-functions/executable.md b/docs/en/sql-reference/table-functions/executable.md
index c6aba61aedb..d377c5d4d0c 100644
--- a/docs/en/sql-reference/table-functions/executable.md
+++ b/docs/en/sql-reference/table-functions/executable.md
@@ -1,6 +1,6 @@
---
slug: /en/engines/table-functions/executable
-sidebar_position: 55
+sidebar_position: 50
sidebar_label: executable
keywords: [udf, user defined function, clickhouse, executable, table, function]
---
diff --git a/docs/en/sql-reference/table-functions/file.md b/docs/en/sql-reference/table-functions/file.md
index f25da96fddb..00917414e0c 100644
--- a/docs/en/sql-reference/table-functions/file.md
+++ b/docs/en/sql-reference/table-functions/file.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/file
-sidebar_position: 37
+sidebar_position: 60
sidebar_label: file
---
@@ -134,7 +134,7 @@ Multiple path components can have globs. For being processed file must exist and
- `*` — Substitutes any number of any characters except `/` including empty string.
- `?` — Substitutes any single character.
-- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`.
+- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`, including `/`.
- `{N..M}` — Substitutes any number in range from N to M including both borders.
- `**` - Fetches all files inside the folder recursively.
diff --git a/docs/en/sql-reference/table-functions/format.md b/docs/en/sql-reference/table-functions/format.md
index 2813eef5bcf..dcebdf16387 100644
--- a/docs/en/sql-reference/table-functions/format.md
+++ b/docs/en/sql-reference/table-functions/format.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/format
-sidebar_position: 56
+sidebar_position: 65
sidebar_label: format
---
diff --git a/docs/en/sql-reference/table-functions/gcs.md b/docs/en/sql-reference/table-functions/gcs.md
index 8574f3ecb9c..01b4e4f6a69 100644
--- a/docs/en/sql-reference/table-functions/gcs.md
+++ b/docs/en/sql-reference/table-functions/gcs.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/gcs
-sidebar_position: 45
+sidebar_position: 70
sidebar_label: gcs
keywords: [gcs, bucket]
---
@@ -16,7 +16,7 @@ gcs(path [,hmac_key, hmac_secret] [,format] [,structure] [,compression])
```
:::tip GCS
-The GCS Table Function integrates with Google Cloud Storage by using the GCS XML API and HMAC keys. See the [Google interoperability docs]( https://cloud.google.com/storage/docs/interoperability) for more details about the endpoint and HMAC.
+The GCS Table Function integrates with Google Cloud Storage by using the GCS XML API and HMAC keys. See the [Google interoperability docs]( https://cloud.google.com/storage/docs/interoperability) for more details about the endpoint and HMAC.
:::
diff --git a/docs/en/sql-reference/table-functions/generate.md b/docs/en/sql-reference/table-functions/generate.md
index 724f6d4a1f2..3b9b077af49 100644
--- a/docs/en/sql-reference/table-functions/generate.md
+++ b/docs/en/sql-reference/table-functions/generate.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/generate
-sidebar_position: 47
+sidebar_position: 75
sidebar_label: generateRandom
---
diff --git a/docs/en/sql-reference/table-functions/hdfs.md b/docs/en/sql-reference/table-functions/hdfs.md
index 1b52e786de4..680ac54ee78 100644
--- a/docs/en/sql-reference/table-functions/hdfs.md
+++ b/docs/en/sql-reference/table-functions/hdfs.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/hdfs
-sidebar_position: 45
+sidebar_position: 80
sidebar_label: hdfs
---
@@ -79,7 +79,7 @@ SELECT count(*)
FROM hdfs('hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV', 'name String, value UInt32')
```
-:::note
+:::note
If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
:::
diff --git a/docs/en/sql-reference/table-functions/hdfsCluster.md b/docs/en/sql-reference/table-functions/hdfsCluster.md
index afd1fd28a5a..832be46d05f 100644
--- a/docs/en/sql-reference/table-functions/hdfsCluster.md
+++ b/docs/en/sql-reference/table-functions/hdfsCluster.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/hdfsCluster
-sidebar_position: 55
+sidebar_position: 81
sidebar_label: hdfsCluster
---
@@ -50,7 +50,7 @@ SELECT count(*)
FROM hdfsCluster('cluster_simple', 'hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV', 'name String, value UInt32')
```
-:::note
+:::note
If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
:::
diff --git a/docs/en/sql-reference/table-functions/hudi.md b/docs/en/sql-reference/table-functions/hudi.md
index 5a97b2401b4..959a32fe26d 100644
--- a/docs/en/sql-reference/table-functions/hudi.md
+++ b/docs/en/sql-reference/table-functions/hudi.md
@@ -1,6 +1,7 @@
---
slug: /en/sql-reference/table-functions/hudi
-sidebar_label: Hudi
+sidebar_position: 85
+sidebar_label: hudi
---
# hudi Table Function
diff --git a/docs/en/sql-reference/table-functions/iceberg.md b/docs/en/sql-reference/table-functions/iceberg.md
index 713b0f9bbf5..30db0ef00aa 100644
--- a/docs/en/sql-reference/table-functions/iceberg.md
+++ b/docs/en/sql-reference/table-functions/iceberg.md
@@ -1,6 +1,7 @@
---
slug: /en/sql-reference/table-functions/iceberg
-sidebar_label: Iceberg
+sidebar_position: 90
+sidebar_label: iceberg
---
# iceberg Table Function
diff --git a/docs/en/sql-reference/table-functions/index.md b/docs/en/sql-reference/table-functions/index.md
index b16295db36a..e8eb983b774 100644
--- a/docs/en/sql-reference/table-functions/index.md
+++ b/docs/en/sql-reference/table-functions/index.md
@@ -1,10 +1,10 @@
---
slug: /en/sql-reference/table-functions/
sidebar_label: Table Functions
-sidebar_position: 34
+sidebar_position: 1
---
-# Table Functions
+# Table Functions
Table functions are methods for constructing tables.
diff --git a/docs/en/sql-reference/table-functions/input.md b/docs/en/sql-reference/table-functions/input.md
index 6aa1cab00c1..1541177b990 100644
--- a/docs/en/sql-reference/table-functions/input.md
+++ b/docs/en/sql-reference/table-functions/input.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/input
-sidebar_position: 46
+sidebar_position: 95
sidebar_label: input
---
diff --git a/docs/en/sql-reference/table-functions/jdbc.md b/docs/en/sql-reference/table-functions/jdbc.md
index 1c12dba9c2b..fbc917c1e1a 100644
--- a/docs/en/sql-reference/table-functions/jdbc.md
+++ b/docs/en/sql-reference/table-functions/jdbc.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/jdbc
-sidebar_position: 43
+sidebar_position: 100
sidebar_label: jdbc
---
diff --git a/docs/en/sql-reference/table-functions/merge.md b/docs/en/sql-reference/table-functions/merge.md
index ba0d19b804e..a1f376ba0eb 100644
--- a/docs/en/sql-reference/table-functions/merge.md
+++ b/docs/en/sql-reference/table-functions/merge.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/merge
-sidebar_position: 38
+sidebar_position: 130
sidebar_label: merge
---
@@ -16,7 +16,7 @@ merge('db_name', 'tables_regexp')
**Arguments**
- `db_name` — Possible values:
- - database name,
+ - database name,
- constant expression that returns a string with a database name, for example, `currentDatabase()`,
- `REGEXP(expression)`, where `expression` is a regular expression to match the DB names.
diff --git a/docs/en/sql-reference/table-functions/mongodb.md b/docs/en/sql-reference/table-functions/mongodb.md
index 042225dd1f0..a483414c0d4 100644
--- a/docs/en/sql-reference/table-functions/mongodb.md
+++ b/docs/en/sql-reference/table-functions/mongodb.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/mongodb
-sidebar_position: 42
+sidebar_position: 135
sidebar_label: mongodb
---
@@ -30,6 +30,14 @@ mongodb(host:port, database, collection, user, password, structure [, options])
- `options` - MongoDB connection string options (optional parameter).
+:::tip
+If you are using the MongoDB Atlas cloud offering please add these options:
+
+```
+'connectTimeoutMS=10000&ssl=true&authSource=admin'
+```
+
+:::
**Returned Value**
diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md
index 269a50ec8b7..0e5b0f54d1c 100644
--- a/docs/en/sql-reference/table-functions/mysql.md
+++ b/docs/en/sql-reference/table-functions/mysql.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/mysql
-sidebar_position: 42
+sidebar_position: 137
sidebar_label: mysql
---
diff --git a/docs/en/sql-reference/table-functions/null.md b/docs/en/sql-reference/table-functions/null.md
index d27295f1916..76e9c32cdbb 100644
--- a/docs/en/sql-reference/table-functions/null.md
+++ b/docs/en/sql-reference/table-functions/null.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/null
-sidebar_position: 53
+sidebar_position: 140
sidebar_label: null function
title: 'null'
---
diff --git a/docs/en/sql-reference/table-functions/numbers.md b/docs/en/sql-reference/table-functions/numbers.md
index a7e49be44a1..32f51363a0a 100644
--- a/docs/en/sql-reference/table-functions/numbers.md
+++ b/docs/en/sql-reference/table-functions/numbers.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/numbers
-sidebar_position: 39
+sidebar_position: 145
sidebar_label: numbers
---
diff --git a/docs/en/sql-reference/table-functions/odbc.md b/docs/en/sql-reference/table-functions/odbc.md
index 781ebacc680..fe6e5390887 100644
--- a/docs/en/sql-reference/table-functions/odbc.md
+++ b/docs/en/sql-reference/table-functions/odbc.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/odbc
-sidebar_position: 44
+sidebar_position: 150
sidebar_label: odbc
---
diff --git a/docs/en/sql-reference/table-functions/postgresql.md b/docs/en/sql-reference/table-functions/postgresql.md
index 3e147fb8417..b9211d70cdb 100644
--- a/docs/en/sql-reference/table-functions/postgresql.md
+++ b/docs/en/sql-reference/table-functions/postgresql.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/postgresql
-sidebar_position: 42
+sidebar_position: 160
sidebar_label: postgresql
---
diff --git a/docs/en/sql-reference/table-functions/redis.md b/docs/en/sql-reference/table-functions/redis.md
index 3efbe3520a7..98d9a647cee 100644
--- a/docs/en/sql-reference/table-functions/redis.md
+++ b/docs/en/sql-reference/table-functions/redis.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/redis
-sidebar_position: 43
+sidebar_position: 170
sidebar_label: redis
---
@@ -31,7 +31,7 @@ redis(host:port, key, structure[, db_index[, password[, pool_size]]])
- `primary` must be specified, it supports only one column in the primary key. The primary key will be serialized in binary as a Redis key.
- columns other than the primary key will be serialized in binary as Redis value in corresponding order.
-
+
- queries with key equals or in filtering will be optimized to multi keys lookup from Redis. If queries without filtering key full table scan will happen which is a heavy operation.
diff --git a/docs/en/sql-reference/table-functions/remote.md b/docs/en/sql-reference/table-functions/remote.md
index ae0ce7bf6d2..59ed4bf1985 100644
--- a/docs/en/sql-reference/table-functions/remote.md
+++ b/docs/en/sql-reference/table-functions/remote.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/remote
-sidebar_position: 40
+sidebar_position: 175
sidebar_label: remote
---
@@ -91,10 +91,10 @@ SELECT * FROM remote_table;
```
### Migration of tables from one system to another:
-This example uses one table from a sample dataset. The database is `imdb`, and the table is `actors`.
+This example uses one table from a sample dataset. The database is `imdb`, and the table is `actors`.
#### On the source ClickHouse system (the system that currently hosts the data)
-- Verify the source database and table name (`imdb.actors`)
+- Verify the source database and table name (`imdb.actors`)
```sql
show databases
```
@@ -116,9 +116,8 @@ This example uses one table from a sample dataset. The database is `imdb`, and
`first_name` String,
`last_name` String,
`gender` FixedString(1))
- ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
- ORDER BY (id, first_name, last_name, gender)
- SETTINGS index_granularity = 8192
+ ENGINE = MergeTree
+ ORDER BY (id, first_name, last_name, gender);
```
#### On the destination ClickHouse system:
@@ -134,9 +133,8 @@ This example uses one table from a sample dataset. The database is `imdb`, and
`first_name` String,
`last_name` String,
`gender` FixedString(1))
- ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
- ORDER BY (id, first_name, last_name, gender)
- SETTINGS index_granularity = 8192
+ ENGINE = MergeTree
+ ORDER BY (id, first_name, last_name, gender);
```
#### Back on the source deployment:
@@ -144,7 +142,7 @@ This example uses one table from a sample dataset. The database is `imdb`, and
Insert into the new database and table created on the remote system. You will need the host, port, username, password, destination database, and destination table.
```sql
INSERT INTO FUNCTION
-remoteSecure('remote.clickhouse.cloud:9440', 'imdb.actors', 'USER', 'PASSWORD', rand())
+remoteSecure('remote.clickhouse.cloud:9440', 'imdb.actors', 'USER', 'PASSWORD')
SELECT * from imdb.actors
```
diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md
index 7068c208022..55c825b8b9b 100644
--- a/docs/en/sql-reference/table-functions/s3.md
+++ b/docs/en/sql-reference/table-functions/s3.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/s3
-sidebar_position: 45
+sidebar_position: 180
sidebar_label: s3
keywords: [s3, gcs, bucket]
---
@@ -33,7 +33,7 @@ For GCS, substitute your HMAC key and HMAC secret where you see `aws_access_key_
and not ~~https://storage.cloud.google.com~~.
:::
-- `NOSIGN` - If this keyword is provided in place of credentials, all the requests will not be signed.
+- `NOSIGN` - If this keyword is provided in place of credentials, all the requests will not be signed.
- `format` — The [format](../../interfaces/formats.md#formats) of the file.
- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`.
- `compression` — Parameter is optional. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. By default, it will autodetect compression by file extension.
diff --git a/docs/en/sql-reference/table-functions/s3Cluster.md b/docs/en/sql-reference/table-functions/s3Cluster.md
index a1d9b9cdad4..d5bdc85f9f8 100644
--- a/docs/en/sql-reference/table-functions/s3Cluster.md
+++ b/docs/en/sql-reference/table-functions/s3Cluster.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/s3Cluster
-sidebar_position: 55
+sidebar_position: 181
sidebar_label: s3Cluster
title: "s3Cluster Table Function"
---
@@ -31,18 +31,18 @@ Select the data from all the files in the `/root/data/clickhouse` and `/root/dat
``` sql
SELECT * FROM s3Cluster(
- 'cluster_simple',
- 'http://minio1:9001/root/data/{clickhouse,database}/*',
- 'minio',
- 'minio123',
- 'CSV',
+ 'cluster_simple',
+ 'http://minio1:9001/root/data/{clickhouse,database}/*',
+ 'minio',
+ 'minio123',
+ 'CSV',
'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))'
) ORDER BY (name, value, polygon);
```
Count the total amount of rows in all files in the cluster `cluster_simple`:
-:::tip
+:::tip
If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
:::
diff --git a/docs/en/sql-reference/table-functions/sqlite.md b/docs/en/sql-reference/table-functions/sqlite.md
index 344fab4fad2..4188b598cb2 100644
--- a/docs/en/sql-reference/table-functions/sqlite.md
+++ b/docs/en/sql-reference/table-functions/sqlite.md
@@ -1,19 +1,19 @@
---
slug: /en/sql-reference/table-functions/sqlite
-sidebar_position: 55
+sidebar_position: 185
sidebar_label: sqlite
title: sqlite
---
Allows to perform queries on a data stored in an [SQLite](../../engines/database-engines/sqlite.md) database.
-**Syntax**
+**Syntax**
``` sql
sqlite('db_path', 'table_name')
```
-**Arguments**
+**Arguments**
- `db_path` — Path to a file with an SQLite database. [String](../../sql-reference/data-types/string.md).
- `table_name` — Name of a table in the SQLite database. [String](../../sql-reference/data-types/string.md).
@@ -40,6 +40,6 @@ Result:
└───────┴──────┘
```
-**See Also**
+**See Also**
- [SQLite](../../engines/table-engines/integrations/sqlite.md) table engine
diff --git a/docs/en/sql-reference/table-functions/url.md b/docs/en/sql-reference/table-functions/url.md
index ac4162c15de..677ed011960 100644
--- a/docs/en/sql-reference/table-functions/url.md
+++ b/docs/en/sql-reference/table-functions/url.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/url
-sidebar_position: 41
+sidebar_position: 200
sidebar_label: url
---
@@ -56,6 +56,7 @@ Character `|` inside patterns is used to specify failover addresses. They are it
## Storage Settings {#storage-settings}
- [engine_url_skip_empty_files](/docs/en/operations/settings/settings.md#engine_url_skip_empty_files) - allows to skip empty files while reading. Disabled by default.
+- [disable_url_encoding](/docs/en/operations/settings/settings.md#disable_url_encoding) - allows to disable decoding/encoding path in uri. Disabled by default.
**See Also**
diff --git a/docs/en/sql-reference/table-functions/urlCluster.md b/docs/en/sql-reference/table-functions/urlCluster.md
index 07d3f4a7362..cf05189112e 100644
--- a/docs/en/sql-reference/table-functions/urlCluster.md
+++ b/docs/en/sql-reference/table-functions/urlCluster.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/urlCluster
-sidebar_position: 55
+sidebar_position: 201
sidebar_label: urlCluster
---
diff --git a/docs/en/sql-reference/table-functions/view.md b/docs/en/sql-reference/table-functions/view.md
index 2c21fe9ff4b..fafb204f31a 100644
--- a/docs/en/sql-reference/table-functions/view.md
+++ b/docs/en/sql-reference/table-functions/view.md
@@ -1,6 +1,6 @@
---
slug: /en/sql-reference/table-functions/view
-sidebar_position: 51
+sidebar_position: 210
sidebar_label: view
title: view
---
diff --git a/docs/en/sql-reference/transactions.md b/docs/en/sql-reference/transactions.md
index 68fbfe0b22a..cb89a091d68 100644
--- a/docs/en/sql-reference/transactions.md
+++ b/docs/en/sql-reference/transactions.md
@@ -3,23 +3,46 @@ slug: /en/guides/developer/transactional
---
# Transactional (ACID) support
-INSERT into one partition* in one table* of MergeTree* family up to max_insert_block_size rows* is transactional (ACID):
-- Atomic: INSERT is succeeded or rejected as a whole: if confirmation is sent to the client, all rows INSERTed; if error is sent to the client, no rows INSERTed.
+## Case 1: INSERT into one partition, of one table, of the MergeTree* family
+
+This is transactional (ACID) if the inserted rows are packed and inserted as a single block (see Notes):
+- Atomic: an INSERT succeeds or is rejected as a whole: if a confirmation is sent to the client, then all rows were inserted; if an error is sent to the client, then no rows were inserted.
- Consistent: if there are no table constraints violated, then all rows in an INSERT are inserted and the INSERT succeeds; if constraints are violated, then no rows are inserted.
-- Isolated: concurrent clients observe a consistent snapshot of the table–the state of the table either as if before INSERT or after successful INSERT; no partial state is seen;
-- Durable: successful INSERT is written to the filesystem before answering to the client, on single replica or multiple replicas (controlled by the `insert_quorum` setting), and ClickHouse can ask the OS to sync the filesystem data on the storage media (controlled by the `fsync_after_insert` setting).
-* If table has many partitions and INSERT covers many partitions–then insertion into every partition is transactional on its own;
-* INSERT into multiple tables with one statement is possible if materialized views are involved;
-* INSERT into Distributed table is not transactional as a whole, while insertion into every shard is transactional;
-* another example: insert into Buffer tables is neither atomic nor isolated or consistent or durable;
-* atomicity is ensured even if `async_insert` is enabled, but it can be turned off by the wait_for_async_insert setting;
-* max_insert_block_size is 1 000 000 by default and can be adjusted as needed;
-* if client did not receive the answer from the server, the client does not know if transaction succeeded, and it can repeat the transaction, using exactly-once insertion properties;
-* ClickHouse is using MVCC with snapshot isolation internally;
-* all ACID properties are valid even in case of server kill / crash;
-* either insert_quorum into different AZ or fsync should be enabled to ensure durable inserts in typical setup;
-* "consistency" in ACID terms does not cover the semantics of distributed systems, see https://jepsen.io/consistency which is controlled by different settings (select_sequential_consistency)
-* this explanation does not cover a new transactions feature that allow to have full-featured transactions over multiple tables, materialized views, for multiple SELECTs, etc.
+- Isolated: concurrent clients observe a consistent snapshot of the table–the state of the table either as it was before the INSERT attempt, or after the successful INSERT; no partial state is seen
+- Durable: a successful INSERT is written to the filesystem before answering to the client, on a single replica or multiple replicas (controlled by the `insert_quorum` setting), and ClickHouse can ask the OS to sync the filesystem data on the storage media (controlled by the `fsync_after_insert` setting).
+- INSERT into multiple tables with one statement is possible if materialized views are involved (the INSERT from the client is to a table which has associate materialized views).
+
+## Case 2: INSERT into multiple partitions, of one table, of the MergeTree* family
+
+Same as Case 1 above, with this detail:
+- If table has many partitions and INSERT covers many partitions–then insertion into every partition is transactional on its own
+
+
+## Case 3: INSERT into one distributed table of the MergeTree* family
+
+Same as Case 1 above, with this detail:
+- INSERT into Distributed table is not transactional as a whole, while insertion into every shard is transactional
+
+## Case 4: Using a Buffer table
+
+- insert into Buffer tables is neither atomic nor isolated nor consistent nor durable
+
+## Case 5: Using async_insert
+
+Same as Case 1 above, with this detail:
+- atomicity is ensured even if `async_insert` is enabled and `wait_for_async_insert` is set to 1 (the default), but if `wait_for_async_insert` is set to 0, then atomicity is not ensured.
+
+## Notes
+- rows inserted from the client in some data format are packed into a single block when:
+ - the insert format is row-based (like CSV, TSV, Values, JSONEachRow, etc) and the data contains less then `max_insert_block_size` rows (~1 000 000 by default) or less then `min_chunk_bytes_for_parallel_parsing` bytes (10 MB by default) in case of parallel parsing is used (enabled by default)
+ - the insert format is column-based (like Native, Parquet, ORC, etc) and the data contains only one block of data
+- the size of the inserted block in general may depend on many settings (for example: `max_block_size`, `max_insert_block_size`, `min_insert_block_size_rows`, `min_insert_block_size_bytes`, `preferred_block_size_bytes`, etc)
+- if the client did not receive an answer from the server, the client does not know if the transaction succeeded, and it can repeat the transaction, using exactly-once insertion properties
+- ClickHouse is using MVCC with snapshot isolation internally
+- all ACID properties are valid even in the case of server kill/crash
+- either insert_quorum into different AZ or fsync should be enabled to ensure durable inserts in the typical setup
+- "consistency" in ACID terms does not cover the semantics of distributed systems, see https://jepsen.io/consistency which is controlled by different settings (select_sequential_consistency)
+- this explanation does not cover a new transactions feature that allow to have full-featured transactions over multiple tables, materialized views, for multiple SELECTs, etc. (see the next section on Transactions, Commit, and Rollback)
## Transactions, Commit, and Rollback
diff --git a/docs/redirects.txt b/docs/redirects.txt
index cea138f7237..ebeda125e01 100644
--- a/docs/redirects.txt
+++ b/docs/redirects.txt
@@ -1,453 +1,6 @@
-agg_functions/combinators.md query-language/agg-functions/combinators.md
-agg_functions/index.md query-language/agg-functions/index.md
-agg_functions/parametric_functions.md query-language/agg-functions/parametric-functions.md
-agg_functions/reference.md query-language/agg-functions/reference.md
-changelog/2017.md whats-new/changelog/2017.md
-changelog/2018.md whats-new/changelog/2018.md
-changelog/2019.md whats-new/changelog/2019.md
-changelog/index.md whats-new/changelog/index.md
-commercial/cloud.md https://clickhouse.com/cloud/
-data_types/array.md sql-reference/data-types/array.md
-data_types/boolean.md sql-reference/data-types/boolean.md
-data_types/date.md sql-reference/data-types/date.md
-data_types/datetime.md sql-reference/data-types/datetime.md
-data_types/datetime64.md sql-reference/data-types/datetime64.md
-data_types/decimal.md sql-reference/data-types/decimal.md
-data_types/domains/ipv4.md sql-reference/data-types/domains/ipv4.md
-data_types/domains/ipv6.md sql-reference/data-types/domains/ipv6.md
-data_types/domains/overview.md sql-reference/data-types/domains/overview.md
-data_types/enum.md sql-reference/data-types/enum.md
-data_types/fixedstring.md sql-reference/data-types/fixedstring.md
-data_types/float.md sql-reference/data-types/float.md
-data_types/index.md sql-reference/data-types/index.md
-data_types/int_uint.md sql-reference/data-types/int-uint.md
-data_types/nested_data_structures/aggregatefunction.md sql-reference/data-types/aggregatefunction.md
-data_types/nested_data_structures/index.md sql-reference/data-types/nested-data-structures/index.md
-data_types/nested_data_structures/nested.md sql-reference/data-types/nested-data-structures/nested.md
-data_types/nullable.md sql-reference/data-types/nullable.md
-data_types/special_data_types/expression.md sql-reference/data-types/special-data-types/expression.md
-data_types/special_data_types/index.md sql-reference/data-types/special-data-types/index.md
-data_types/special_data_types/interval.md sql-reference/data-types/special-data-types/interval.md
-data_types/special_data_types/nothing.md sql-reference/data-types/special-data-types/nothing.md
-data_types/special_data_types/set.md sql-reference/data-types/special-data-types/set.md
-data_types/string.md sql-reference/data-types/string.md
-data_types/tuple.md sql-reference/data-types/tuple.md
-data_types/uuid.md sql-reference/data-types/uuid.md
-database_engines/index.md engines/database-engines/index.md
-database_engines/lazy.md engines/database-engines/lazy.md
-database_engines/mysql.md engines/database-engines/mysql.md
-development/browse_code.md development/browse-code.md
-development/build_cross_arm.md development/build-cross-arm.md
-development/build_cross_osx.md development/build-cross-osx.md
-development/build_osx.md development/build-osx.md
-development/developer_instruction.md development/developer-instruction.md
-dicts/external_dicts.md query-language/dicts/external-dicts.md
-dicts/external_dicts_dict.md query-language/dicts/external-dicts-dict.md
-dicts/external_dicts_dict_layout.md query-language/dicts/external-dicts-dict-layout.md
-dicts/external_dicts_dict_lifetime.md query-language/dicts/external-dicts-dict-lifetime.md
-dicts/external_dicts_dict_sources.md query-language/dicts/external-dicts-dict-sources.md
-dicts/external_dicts_dict_structure.md query-language/dicts/external-dicts-dict-structure.md
-dicts/index.md query-language/dicts/index.md
-dicts/internal_dicts.md query-language/dicts/internal-dicts.md
-engines/database_engines/index.md engines/database-engines/index.md
-engines/database_engines/lazy.md engines/database-engines/lazy.md
-engines/database_engines/mysql.md engines/database-engines/mysql.md
-engines/table-engines/log-family/log-family.md engines/table-engines/log-family/index.md
-engines/table_engines/index.md engines/table-engines/index.md
-engines/table_engines/integrations/hdfs.md engines/table-engines/integrations/hdfs.md
-engines/table_engines/integrations/index.md engines/table-engines/integrations/index.md
-engines/table_engines/integrations/jdbc.md engines/table-engines/integrations/jdbc.md
-engines/table_engines/integrations/kafka.md engines/table-engines/integrations/kafka.md
-engines/table_engines/integrations/mysql.md engines/table-engines/integrations/mysql.md
-engines/table_engines/integrations/odbc.md engines/table-engines/integrations/odbc.md
-engines/table_engines/log_family/index.md engines/table-engines/log-family/index.md
-engines/table_engines/log_family/log.md engines/table-engines/log-family/log.md
-engines/table_engines/log_family/log_family.md engines/table-engines/log-family/log-family.md
-engines/table_engines/log_family/stripelog.md engines/table-engines/log-family/stripelog.md
-engines/table_engines/log_family/tinylog.md engines/table-engines/log-family/tinylog.md
-engines/table_engines/mergetree_family/aggregatingmergetree.md engines/table-engines/mergetree-family/aggregatingmergetree.md
-engines/table_engines/mergetree_family/collapsingmergetree.md engines/table-engines/mergetree-family/collapsingmergetree.md
-engines/table_engines/mergetree_family/custom_partitioning_key.md engines/table-engines/mergetree-family/custom-partitioning-key.md
-engines/table_engines/mergetree_family/graphitemergetree.md engines/table-engines/mergetree-family/graphitemergetree.md
-engines/table_engines/mergetree_family/index.md engines/table-engines/mergetree-family/index.md
-engines/table_engines/mergetree_family/mergetree.md engines/table-engines/mergetree-family/mergetree.md
-engines/table_engines/mergetree_family/replacingmergetree.md engines/table-engines/mergetree-family/replacingmergetree.md
-engines/table_engines/mergetree_family/replication.md engines/table-engines/mergetree-family/replication.md
-engines/table_engines/mergetree_family/summingmergetree.md engines/table-engines/mergetree-family/summingmergetree.md
-engines/table_engines/mergetree_family/versionedcollapsingmergetree.md engines/table-engines/mergetree-family/versionedcollapsingmergetree.md
-engines/table_engines/special/buffer.md engines/table-engines/special/buffer.md
-engines/table_engines/special/dictionary.md engines/table-engines/special/dictionary.md
-engines/table_engines/special/distributed.md engines/table-engines/special/distributed.md
-engines/table_engines/special/external_data.md engines/table-engines/special/external-data.md
-engines/table_engines/special/file.md engines/table-engines/special/file.md
-engines/table_engines/special/generate.md engines/table-engines/special/generate.md
-engines/table_engines/special/index.md engines/table-engines/special/index.md
-engines/table_engines/special/join.md engines/table-engines/special/join.md
-engines/table_engines/special/materializedview.md engines/table-engines/special/materializedview.md
-engines/table_engines/special/memory.md engines/table-engines/special/memory.md
-engines/table_engines/special/merge.md engines/table-engines/special/merge.md
-engines/table_engines/special/null.md engines/table-engines/special/null.md
-engines/table_engines/special/set.md engines/table-engines/special/set.md
-engines/table_engines/special/url.md engines/table-engines/special/url.md
-engines/table_engines/special/view.md engines/table-engines/special/view.md
-extended_roadmap.md whats-new/extended-roadmap.md
-formats.md interfaces/formats.md
-formats/capnproto.md interfaces/formats.md
-formats/csv.md interfaces/formats.md
-formats/csvwithnames.md interfaces/formats.md
-formats/json.md interfaces/formats.md
-formats/jsoncompact.md interfaces/formats.md
-formats/jsoneachrow.md interfaces/formats.md
-formats/native.md interfaces/formats.md
-formats/null.md interfaces/formats.md
-formats/pretty.md interfaces/formats.md
-formats/prettycompact.md interfaces/formats.md
-formats/prettycompactmonoblock.md interfaces/formats.md
-formats/prettynoescapes.md interfaces/formats.md
-formats/prettyspace.md interfaces/formats.md
-formats/rowbinary.md interfaces/formats.md
-formats/tabseparated.md interfaces/formats.md
-formats/tabseparatedraw.md interfaces/formats.md
-formats/tabseparatedwithnames.md interfaces/formats.md
-formats/tabseparatedwithnamesandtypes.md interfaces/formats.md
-formats/tskv.md interfaces/formats.md
-formats/values.md interfaces/formats.md
-formats/vertical.md interfaces/formats.md
-formats/verticalraw.md interfaces/formats.md
-formats/xml.md interfaces/formats.md
-functions/arithmetic_functions.md query-language/functions/arithmetic-functions.md
-functions/array_functions.md query-language/functions/array-functions.md
-functions/array_join.md query-language/functions/array-join.md
-functions/bit_functions.md query-language/functions/bit-functions.md
-functions/bitmap_functions.md query-language/functions/bitmap-functions.md
-functions/comparison_functions.md query-language/functions/comparison-functions.md
-functions/conditional_functions.md query-language/functions/conditional-functions.md
-functions/date_time_functions.md query-language/functions/date-time-functions.md
-functions/encoding_functions.md query-language/functions/encoding-functions.md
-functions/ext_dict_functions.md query-language/functions/ext-dict-functions.md
-functions/hash_functions.md query-language/functions/hash-functions.md
-functions/higher_order_functions.md query-language/functions/higher-order-functions.md
-functions/in_functions.md query-language/functions/in-functions.md
-functions/index.md query-language/functions/index.md
-functions/ip_address_functions.md query-language/functions/ip-address-functions.md
-functions/json_functions.md query-language/functions/json-functions.md
-functions/logical_functions.md query-language/functions/logical-functions.md
-functions/math_functions.md query-language/functions/math-functions.md
-functions/other_functions.md query-language/functions/other-functions.md
-functions/random_functions.md query-language/functions/random-functions.md
-functions/rounding_functions.md query-language/functions/rounding-functions.md
-functions/splitting_merging_functions.md query-language/functions/splitting-merging-functions.md
-functions/string_functions.md query-language/functions/string-functions.md
-functions/string_replace_functions.md query-language/functions/string-replace-functions.md
-functions/string_search_functions.md query-language/functions/string-search-functions.md
-functions/type_conversion_functions.md query-language/functions/type-conversion-functions.md
-functions/url_functions.md query-language/functions/url-functions.md
-functions/ym_dict_functions.md query-language/functions/ym-dict-functions.md
-getting_started/example_datasets/amplab_benchmark.md getting-started/example-datasets/amplab-benchmark.md
-getting_started/example_datasets/criteo.md getting-started/example-datasets/criteo.md
-getting_started/example_datasets/index.md getting-started/example-datasets/index.md
-getting_started/example_datasets/metrica.md getting-started/example-datasets/metrica.md
-getting_started/example_datasets/nyc_taxi.md getting-started/example-datasets/nyc-taxi.md
-getting_started/example_datasets/ontime.md getting-started/example-datasets/ontime.md
-getting_started/example_datasets/star_schema.md getting-started/example-datasets/star-schema.md
-getting_started/example_datasets/wikistat.md getting-started/example-datasets/wikistat.md
-getting_started/index.md getting-started/index.md
-getting_started/install.md getting-started/install.md
-getting_started/playground.md getting-started/playground.md
-getting_started/tutorial.md getting-started/tutorial.md
-images/column_oriented.gif images/column-oriented.gif
-images/row_oriented.gif images/row-oriented.gif
-interfaces/http_interface.md interfaces/http.md
-interfaces/third-party/client_libraries.md interfaces/third-party/client-libraries.md
-interfaces/third-party_client_libraries.md interfaces/third-party/client-libraries.md
-interfaces/third-party_gui.md interfaces/third-party/gui.md
-interfaces/third_party/index.md interfaces/third-party/index.md
-introduction/index.md
-introduction/distinctive_features.md introduction/distinctive-features.md
-introduction/features_considered_disadvantages.md introduction/distinctive-features.md
-introduction/possible_silly_questions.md faq/general.md
-introduction/ya_metrika_task.md introduction/history.md
-operations/access_rights.md operations/access-rights.md
-operations/configuration_files.md operations/configuration-files.md
-operations/optimizing_performance/index.md operations/optimizing-performance/index.md
-operations/optimizing_performance/sampling_query_profiler.md operations/optimizing-performance/sampling-query-profiler.md
-operations/performance/sampling_query_profiler.md operations/optimizing-performance/sampling-query-profiler.md
-operations/performance_test.md operations/performance-test.md
-operations/server_configuration_parameters/index.md operations/server-configuration-parameters/index.md
-operations/server_configuration_parameters/settings.md operations/server-configuration-parameters/settings.md
-operations/server_settings/index.md operations/server-configuration-parameters/index.md
-operations/server_settings/settings.md operations/server-configuration-parameters/settings.md
-operations/settings/constraints_on_settings.md operations/settings/constraints-on-settings.md
-operations/settings/permissions_for_queries.md operations/settings/permissions-for-queries.md
-operations/settings/query_complexity.md operations/settings/query-complexity.md
-operations/settings/settings_profiles.md operations/settings/settings-profiles.md
-operations/settings/settings_users.md operations/settings/settings-users.md
-operations/system_tables.md operations/system-tables.md
-operations/table_engines/aggregatingmergetree.md engines/table-engines/mergetree-family/aggregatingmergetree.md
-operations/table_engines/buffer.md engines/table-engines/special/buffer.md
-operations/table_engines/collapsingmergetree.md engines/table-engines/mergetree-family/collapsingmergetree.md
-operations/table_engines/custom_partitioning_key.md engines/table-engines/mergetree-family/custom-partitioning-key.md
-operations/table_engines/dictionary.md engines/table-engines/special/dictionary.md
-operations/table_engines/distributed.md engines/table-engines/special/distributed.md
-operations/table_engines/external_data.md engines/table-engines/special/external-data.md
-operations/table_engines/file.md engines/table-engines/special/file.md
-operations/table_engines/generate.md engines/table-engines/special/generate.md
-operations/table_engines/graphitemergetree.md engines/table-engines/mergetree-family/graphitemergetree.md
-operations/table_engines/hdfs.md engines/table-engines/integrations/hdfs.md
-operations/table_engines/index.md engines/table-engines/index.md
-operations/table_engines/jdbc.md engines/table-engines/integrations/jdbc.md
-operations/table_engines/join.md engines/table-engines/special/join.md
-operations/table_engines/kafka.md engines/table-engines/integrations/kafka.md
-operations/table_engines/log.md engines/table-engines/log-family/log.md
-operations/table_engines/log_family.md engines/table-engines/log-family/log-family.md
-operations/table_engines/materializedview.md engines/table-engines/special/materializedview.md
-operations/table_engines/memory.md engines/table-engines/special/memory.md
-operations/table_engines/merge.md engines/table-engines/special/merge.md
-operations/table_engines/mergetree.md engines/table-engines/mergetree-family/mergetree.md
-operations/table_engines/mysql.md engines/table-engines/integrations/mysql.md
-operations/table_engines/null.md engines/table-engines/special/null.md
-operations/table_engines/odbc.md engines/table-engines/integrations/odbc.md
-operations/table_engines/replacingmergetree.md engines/table-engines/mergetree-family/replacingmergetree.md
-operations/table_engines/replication.md engines/table-engines/mergetree-family/replication.md
-operations/table_engines/set.md engines/table-engines/special/set.md
-operations/table_engines/stripelog.md engines/table-engines/log-family/stripelog.md
-operations/table_engines/summingmergetree.md engines/table-engines/mergetree-family/summingmergetree.md
-operations/table_engines/tinylog.md engines/table-engines/log-family/tinylog.md
-operations/table_engines/url.md engines/table-engines/special/url.md
-operations/table_engines/versionedcollapsingmergetree.md engines/table-engines/mergetree-family/versionedcollapsingmergetree.md
-operations/table_engines/view.md engines/table-engines/special/view.md
-operations/utils/clickhouse-benchmark.md operations/utilities/clickhouse-benchmark.md
-operations/utils/clickhouse-copier.md operations/utilities/clickhouse-copier.md
-operations/utils/clickhouse-local.md operations/utilities/clickhouse-local.md
-operations/utils/index.md operations/utilities/index.md
-query_language/agg_functions/combinators.md sql-reference/aggregate-functions/combinators.md
-query_language/agg_functions/index.md sql-reference/aggregate-functions/index.md
-query_language/agg_functions/parametric_functions.md sql-reference/aggregate-functions/parametric-functions.md
-query_language/agg_functions/reference.md sql-reference/aggregate-functions/reference.md
-query_language/alter.md sql-reference/statements/alter.md
-query_language/create.md sql-reference/statements/create.md
-query_language/dicts/external_dicts.md sql-reference/dictionaries/external-dictionaries/external-dicts.md
-query_language/dicts/external_dicts_dict.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md
-query_language/dicts/external_dicts_dict_hierarchical.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md
-query_language/dicts/external_dicts_dict_layout.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md
-query_language/dicts/external_dicts_dict_lifetime.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md
-query_language/dicts/external_dicts_dict_sources.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md
-query_language/dicts/external_dicts_dict_structure.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md
-query_language/dicts/index.md sql-reference/dictionaries/index.md
-query_language/dicts/internal_dicts.md sql-reference/dictionaries/internal-dicts.md
-query_language/functions/arithmetic_functions.md sql-reference/functions/arithmetic-functions.md
-query_language/functions/array_functions.md sql-reference/functions/array-functions.md
-query_language/functions/array_join.md sql-reference/functions/array-join.md
-query_language/functions/bit_functions.md sql-reference/functions/bit-functions.md
-query_language/functions/bitmap_functions.md sql-reference/functions/bitmap-functions.md
-query_language/functions/comparison_functions.md sql-reference/functions/comparison-functions.md
-query_language/functions/conditional_functions.md sql-reference/functions/conditional-functions.md
-query_language/functions/date_time_functions.md sql-reference/functions/date-time-functions.md
-query_language/functions/encoding_functions.md sql-reference/functions/encoding-functions.md
-query_language/functions/ext_dict_functions.md sql-reference/functions/ext-dict-functions.md
-query_language/functions/functions_for_nulls.md sql-reference/functions/functions-for-nulls.md
-query_language/functions/geo.md sql-reference/functions/geo.md
-query_language/functions/hash_functions.md sql-reference/functions/hash-functions.md
-query_language/functions/higher_order_functions.md sql-reference/functions/higher-order-functions.md
-query_language/functions/in_functions.md sql-reference/functions/in-functions.md
-query_language/functions/index.md sql-reference/functions/index.md
-query_language/functions/introspection.md sql-reference/functions/introspection.md
-query_language/functions/ip_address_functions.md sql-reference/functions/ip-address-functions.md
-query_language/functions/json_functions.md sql-reference/functions/json-functions.md
-query_language/functions/logical_functions.md sql-reference/functions/logical-functions.md
-query_language/functions/machine_learning_functions.md sql-reference/functions/machine-learning-functions.md
-query_language/functions/math_functions.md sql-reference/functions/math-functions.md
-query_language/functions/other_functions.md sql-reference/functions/other-functions.md
-query_language/functions/random_functions.md sql-reference/functions/random-functions.md
-query_language/functions/rounding_functions.md sql-reference/functions/rounding-functions.md
-query_language/functions/splitting_merging_functions.md sql-reference/functions/splitting-merging-functions.md
-query_language/functions/string_functions.md sql-reference/functions/string-functions.md
-query_language/functions/string_replace_functions.md sql-reference/functions/string-replace-functions.md
-query_language/functions/string_search_functions.md sql-reference/functions/string-search-functions.md
-query_language/functions/type_conversion_functions.md sql-reference/functions/type-conversion-functions.md
-query_language/functions/url_functions.md sql-reference/functions/url-functions.md
-query_language/functions/uuid_functions.md sql-reference/functions/uuid-functions.md
-query_language/functions/ym_dict_functions.md sql-reference/functions/ym-dict-functions.md
-query_language/index.md sql-reference/index.md
-query_language/insert_into.md sql-reference/statements/insert-into.md
-query_language/misc.md sql-reference/statements/misc.md
-query_language/operators.md sql-reference/operators.md
-query_language/queries.md query-language.md
-query_language/select.md sql-reference/statements/select.md
-query_language/show.md sql-reference/statements/show.md
-query_language/syntax.md sql-reference/syntax.md
-query_language/system.md sql-reference/statements/system.md
-query_language/table_functions/file.md sql-reference/table-functions/file.md
-query_language/table_functions/generate.md sql-reference/table-functions/generate.md
-query_language/table_functions/hdfs.md sql-reference/table-functions/hdfs.md
-query_language/table_functions/index.md sql-reference/table-functions/index.md
-query_language/table_functions/input.md sql-reference/table-functions/input.md
-query_language/table_functions/jdbc.md sql-reference/table-functions/jdbc.md
-query_language/table_functions/merge.md sql-reference/table-functions/merge.md
-query_language/table_functions/mysql.md sql-reference/table-functions/mysql.md
-query_language/table_functions/numbers.md sql-reference/table-functions/numbers.md
-query_language/table_functions/odbc.md sql-reference/table-functions/odbc.md
-query_language/table_functions/remote.md sql-reference/table-functions/remote.md
-query_language/table_functions/url.md sql-reference/table-functions/url.md
-roadmap.md whats-new/roadmap.md
-security_changelog.md whats-new/security-changelog.md
-sql-reference/data-types/domains/overview.md sql-reference/data-types/domains/index.md
-sql_reference/aggregate_functions/combinators.md sql-reference/aggregate-functions/combinators.md
-sql_reference/aggregate_functions/index.md sql-reference/aggregate-functions/index.md
-sql_reference/aggregate_functions/parametric_functions.md sql-reference/aggregate-functions/parametric-functions.md
-sql_reference/aggregate_functions/reference.md sql-reference/aggregate-functions/reference.md
-sql_reference/ansi.md sql-reference/ansi.md
-sql_reference/data_types/aggregatefunction.md sql-reference/data-types/aggregatefunction.md
-sql_reference/data_types/array.md sql-reference/data-types/array.md
-sql_reference/data_types/boolean.md sql-reference/data-types/boolean.md
-sql_reference/data_types/date.md sql-reference/data-types/date.md
-sql_reference/data_types/datetime.md sql-reference/data-types/datetime.md
-sql_reference/data_types/datetime64.md sql-reference/data-types/datetime64.md
-sql_reference/data_types/decimal.md sql-reference/data-types/decimal.md
-sql_reference/data_types/domains/index.md sql-reference/data-types/domains/index.md
-sql_reference/data_types/domains/ipv4.md sql-reference/data-types/domains/ipv4.md
-sql_reference/data_types/domains/ipv6.md sql-reference/data-types/domains/ipv6.md
-sql_reference/data_types/domains/overview.md sql-reference/data-types/domains/overview.md
-sql_reference/data_types/enum.md sql-reference/data-types/enum.md
-sql_reference/data_types/fixedstring.md sql-reference/data-types/fixedstring.md
-sql_reference/data_types/float.md sql-reference/data-types/float.md
-sql_reference/data_types/index.md sql-reference/data-types/index.md
-sql_reference/data_types/int_uint.md sql-reference/data-types/int-uint.md
-sql_reference/data_types/nested_data_structures/index.md sql-reference/data-types/nested-data-structures/index.md
-sql_reference/data_types/nested_data_structures/nested.md sql-reference/data-types/nested-data-structures/nested.md
-sql_reference/data_types/nullable.md sql-reference/data-types/nullable.md
-sql_reference/data_types/simpleaggregatefunction.md sql-reference/data-types/simpleaggregatefunction.md
-sql_reference/data_types/special_data_types/expression.md sql-reference/data-types/special-data-types/expression.md
-sql_reference/data_types/special_data_types/index.md sql-reference/data-types/special-data-types/index.md
-sql_reference/data_types/special_data_types/interval.md sql-reference/data-types/special-data-types/interval.md
-sql_reference/data_types/special_data_types/nothing.md sql-reference/data-types/special-data-types/nothing.md
-sql_reference/data_types/special_data_types/set.md sql-reference/data-types/special-data-types/set.md
-sql_reference/data_types/string.md sql-reference/data-types/string.md
-sql_reference/data_types/tuple.md sql-reference/data-types/tuple.md
-sql_reference/data_types/uuid.md sql-reference/data-types/uuid.md
-sql_reference/dictionaries/external_dictionaries/external_dicts.md sql-reference/dictionaries/external-dictionaries/external-dicts.md
-sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md
-sql_reference/dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md
-sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md
-sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md
-sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md
-sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md
-sql_reference/dictionaries/external_dictionaries/index.md sql-reference/dictionaries/external-dictionaries/index.md
-sql_reference/dictionaries/index.md sql-reference/dictionaries/index.md
-sql_reference/dictionaries/internal_dicts.md sql-reference/dictionaries/internal-dicts.md
-sql_reference/functions/arithmetic_functions.md sql-reference/functions/arithmetic-functions.md
-sql_reference/functions/array_functions.md sql-reference/functions/array-functions.md
-sql_reference/functions/array_join.md sql-reference/functions/array-join.md
-sql_reference/functions/bit_functions.md sql-reference/functions/bit-functions.md
-sql_reference/functions/bitmap_functions.md sql-reference/functions/bitmap-functions.md
-sql_reference/functions/comparison_functions.md sql-reference/functions/comparison-functions.md
-sql_reference/functions/conditional_functions.md sql-reference/functions/conditional-functions.md
-sql_reference/functions/date_time_functions.md sql-reference/functions/date-time-functions.md
-sql_reference/functions/encoding_functions.md sql-reference/functions/encoding-functions.md
-sql_reference/functions/ext_dict_functions.md sql-reference/functions/ext-dict-functions.md
-sql_reference/functions/functions_for_nulls.md sql-reference/functions/functions-for-nulls.md
-sql_reference/functions/geo.md sql-reference/functions/geo.md
-sql_reference/functions/hash_functions.md sql-reference/functions/hash-functions.md
-sql_reference/functions/higher_order_functions.md sql-reference/functions/higher-order-functions.md
-sql_reference/functions/in_functions.md sql-reference/functions/in-functions.md
-sql_reference/functions/index.md sql-reference/functions/index.md
-sql_reference/functions/introspection.md sql-reference/functions/introspection.md
-sql_reference/functions/ip_address_functions.md sql-reference/functions/ip-address-functions.md
-sql_reference/functions/json_functions.md sql-reference/functions/json-functions.md
-sql_reference/functions/logical_functions.md sql-reference/functions/logical-functions.md
-sql_reference/functions/machine_learning_functions.md sql-reference/functions/machine-learning-functions.md
-sql_reference/functions/math_functions.md sql-reference/functions/math-functions.md
-sql_reference/functions/other_functions.md sql-reference/functions/other-functions.md
-sql_reference/functions/random_functions.md sql-reference/functions/random-functions.md
-sql_reference/functions/rounding_functions.md sql-reference/functions/rounding-functions.md
-sql_reference/functions/splitting_merging_functions.md sql-reference/functions/splitting-merging-functions.md
-sql_reference/functions/string_functions.md sql-reference/functions/string-functions.md
-sql_reference/functions/string_replace_functions.md sql-reference/functions/string-replace-functions.md
-sql_reference/functions/string_search_functions.md sql-reference/functions/string-search-functions.md
-sql_reference/functions/type_conversion_functions.md sql-reference/functions/type-conversion-functions.md
-sql_reference/functions/url_functions.md sql-reference/functions/url-functions.md
-sql_reference/functions/uuid_functions.md sql-reference/functions/uuid-functions.md
-sql_reference/functions/ym_dict_functions.md sql-reference/functions/ym-dict-functions.md
-sql_reference/index.md sql-reference/index.md
-sql_reference/operators.md sql-reference/operators.md
-sql_reference/statements/alter.md sql-reference/statements/alter.md
-sql_reference/statements/create.md sql-reference/statements/create.md
-sql_reference/statements/index.md sql-reference/statements/index.md
-sql_reference/statements/insert_into.md sql-reference/statements/insert-into.md
-sql_reference/statements/misc.md sql-reference/statements/misc.md
-sql_reference/statements/select.md sql-reference/statements/select.md
-sql_reference/statements/show.md sql-reference/statements/show.md
-sql_reference/statements/system.md sql-reference/statements/system.md
-sql_reference/syntax.md sql-reference/syntax.md
-sql_reference/table_functions/file.md sql-reference/table-functions/file.md
-sql_reference/table_functions/generate.md sql-reference/table-functions/generate.md
-sql_reference/table_functions/hdfs.md sql-reference/table-functions/hdfs.md
-sql_reference/table_functions/index.md sql-reference/table-functions/index.md
-sql_reference/table_functions/input.md sql-reference/table-functions/input.md
-sql_reference/table_functions/jdbc.md sql-reference/table-functions/jdbc.md
-sql_reference/table_functions/merge.md sql-reference/table-functions/merge.md
-sql_reference/table_functions/mysql.md sql-reference/table-functions/mysql.md
-sql_reference/table_functions/numbers.md sql-reference/table-functions/numbers.md
-sql_reference/table_functions/odbc.md sql-reference/table-functions/odbc.md
-sql_reference/table_functions/remote.md sql-reference/table-functions/remote.md
-sql_reference/table_functions/url.md sql-reference/table-functions/url.md
-system_tables.md operations/system-tables.md
-system_tables/system.asynchronous_metrics.md operations/system-tables.md
-system_tables/system.clusters.md operations/system-tables.md
-system_tables/system.columns.md operations/system-tables.md
-system_tables/system.databases.md operations/system-tables.md
-system_tables/system.dictionaries.md operations/system-tables.md
-system_tables/system.events.md operations/system-tables.md
-system_tables/system.functions.md operations/system-tables.md
-system_tables/system.merges.md operations/system-tables.md
-system_tables/system.metrics.md operations/system-tables.md
-system_tables/system.numbers.md operations/system-tables.md
-system_tables/system.numbers_mt.md operations/system-tables.md
-system_tables/system.one.md operations/system-tables.md
-system_tables/system.parts.md operations/system-tables.md
-system_tables/system.processes.md operations/system-tables.md
-system_tables/system.replicas.md operations/system-tables.md
-system_tables/system.settings.md operations/system-tables.md
-system_tables/system.tables.md operations/system-tables.md
-system_tables/system.zookeeper.md operations/system-tables.md
-table_engines.md operations/table-engines.md
-table_engines/aggregatingmergetree.md operations/table-engines/aggregatingmergetree.md
-table_engines/buffer.md operations/table-engines/buffer.md
-table_engines/collapsingmergetree.md operations/table-engines/collapsingmergetree.md
-table_engines/custom_partitioning_key.md operations/table-engines/custom-partitioning-key.md
-table_engines/dictionary.md operations/table-engines/dictionary.md
-table_engines/distributed.md operations/table-engines/distributed.md
-table_engines/external_data.md operations/table-engines/external-data.md
-table_engines/file.md operations/table-engines/file.md
-table_engines/graphitemergetree.md operations/table-engines/graphitemergetree.md
-table_engines/index.md operations/table-engines/index.md
-table_engines/join.md operations/table-engines/join.md
-table_engines/kafka.md operations/table-engines/kafka.md
-table_engines/log.md operations/table-engines/log.md
-table_engines/materializedview.md operations/table-engines/materializedview.md
-table_engines/memory.md operations/table-engines/memory.md
-table_engines/merge.md operations/table-engines/merge.md
-table_engines/mergetree.md operations/table-engines/mergetree.md
-table_engines/mysql.md operations/table-engines/mysql.md
-table_engines/null.md operations/table-engines/null.md
-table_engines/replacingmergetree.md operations/table-engines/replacingmergetree.md
-table_engines/replication.md operations/table-engines/replication.md
-table_engines/set.md operations/table-engines/set.md
-table_engines/summingmergetree.md operations/table-engines/summingmergetree.md
-table_engines/tinylog.md operations/table-engines/tinylog.md
-table_engines/view.md operations/table-engines/view.md
-table_functions/file.md query-language/table-functions/file.md
-table_functions/index.md query-language/table-functions/index.md
-table_functions/merge.md query-language/table-functions/merge.md
-table_functions/numbers.md query-language/table-functions/numbers.md
-table_functions/remote.md query-language/table-functions/remote.md
-utils.md operations/utils.md
-utils/clickhouse-copier.md operations/utils/clickhouse-copier.md
-utils/clickhouse-local.md operations/utils/clickhouse-local.md
-whats_new/changelog/2017.md whats-new/changelog/2017.md
-whats_new/changelog/2018.md whats-new/changelog/2018.md
-whats_new/changelog/2019.md whats-new/changelog/2019.md
-whats_new/changelog/index.md whats-new/changelog/index.md
-whats_new/index.md whats-new/index.md
-whats_new/roadmap.md whats-new/roadmap.md
-whats_new/security_changelog.md whats-new/security-changelog.md
+The redirects from this file were moved to the Docusaurus configuration file.
+If you need to add a redirect please either open a PR in
+https://github.com/clickhouse/clickhouse-docs adding the redirect to
+https://github.com/ClickHouse/clickhouse-docs/blob/main/docusaurus.config.js
+or open an issue in the same repo and provide the old URL and new URL to have
+the redirect added.
diff --git a/docs/ru/development/build-osx.md b/docs/ru/development/build-osx.md
index 9a1f9c9347d..6b4e612b13f 100644
--- a/docs/ru/development/build-osx.md
+++ b/docs/ru/development/build-osx.md
@@ -68,7 +68,7 @@ $ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/
$ rm -rf build
$ mkdir build
$ cd build
- $ cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER==$(brew --prefix llvm)/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF ..
+ $ cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF ..
$ cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF ..
$ cmake --build . --config RelWithDebInfo
$ cd ..
diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md
index 48a6132170a..e232b63f049 100644
--- a/docs/ru/interfaces/formats.md
+++ b/docs/ru/interfaces/formats.md
@@ -401,8 +401,8 @@ $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FOR
- [output_format_csv_crlf_end_of_line](../operations/settings/settings.md#output_format_csv_crlf_end_of_line) - если установлено значение true, конец строки в формате вывода CSV будет `\r\n` вместо `\n`. Значение по умолчанию - `false`.
- [input_format_csv_skip_first_lines](../operations/settings/settings.md#input_format_csv_skip_first_lines) - пропустить указанное количество строк в начале данных. Значение по умолчанию - `0`.
- [input_format_csv_detect_header](../operations/settings/settings.md#input_format_csv_detect_header) - обнаружить заголовок с именами и типами в формате CSV. Значение по умолчанию - `true`.
-- [input_format_csv_trim_whitespaces](../operations/settings/settings.md#input_format_csv_trim_whitespaces) - удалить пробелы и символы табуляции из строк без кавычек.
-Значение по умолчанию - `true`.
+- [input_format_csv_trim_whitespaces](../operations/settings/settings.md#input_format_csv_trim_whitespaces) - удалить пробелы и символы табуляции из строк без кавычек. Значение по умолчанию - `true`.
+- [input_format_csv_allow_variable_number_of_columns](../operations/settings/settings.md/#input_format_csv_allow_variable_number_of_columns) - игнорировать дополнительные столбцы (если файл содержит больше столбцов чем ожидается) и рассматривать отсутствующие поля в CSV в качестве значений по умолчанию. Значение по умолчанию - `false`.
## CSVWithNames {#csvwithnames}
diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md
index b8c5ee77f0c..981f1c7b5a2 100644
--- a/docs/ru/interfaces/http.md
+++ b/docs/ru/interfaces/http.md
@@ -50,7 +50,7 @@ Connection: Close
Content-Type: text/tab-separated-values; charset=UTF-8
X-ClickHouse-Server-Display-Name: clickhouse.ru-central1.internal
X-ClickHouse-Query-Id: 5abe861c-239c-467f-b955-8a201abb8b7f
-X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
1
```
@@ -266,9 +266,9 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812
Прогресс выполнения запроса можно отслеживать с помощью заголовков ответа `X-ClickHouse-Progress`. Для этого включите [send_progress_in_http_headers](../operations/settings/settings.md#settings-send_progress_in_http_headers). Пример последовательности заголовков:
``` text
-X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128"}
-X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128"}
-X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128"}
+X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128","peak_memory_usage":"4371480"}
+X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128","peak_memory_usage":"13621616"}
+X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128","peak_memory_usage":"23155600"}
```
Возможные поля заголовка:
@@ -529,7 +529,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
* Connection #0 to host localhost left intact
Say Hi!%
@@ -569,7 +569,7 @@ $ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler'
< Content-Type: text/plain; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
* Connection #0 to host localhost left intact
%
@@ -621,7 +621,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
Absolute Path File
* Connection #0 to host localhost left intact
@@ -640,7 +640,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
Relative Path File
* Connection #0 to host localhost left intact
diff --git a/docs/ru/operations/configuration-files.md b/docs/ru/operations/configuration-files.md
index 2b824ce91bd..01a91bd41c6 100644
--- a/docs/ru/operations/configuration-files.md
+++ b/docs/ru/operations/configuration-files.md
@@ -85,6 +85,40 @@ $ cat /etc/clickhouse-server/users.d/alice.xml
Сервер следит за изменениями конфигурационных файлов, а также файлов и ZooKeeper-узлов, которые были использованы при выполнении подстановок и переопределений, и перезагружает настройки пользователей и кластеров на лету. То есть, можно изменять кластера, пользователей и их настройки без перезапуска сервера.
+## Шифрование {#encryption}
+
+Вы можете использовать симметричное шифрование для зашифровки элемента конфигурации, например, поля password. Чтобы это сделать, сначала настройте [кодек шифрования](../sql-reference/statements/create/table.md#encryption-codecs), затем добавьте аттибут`encryption_codec` с именем кодека шифрования как значение к элементу, который надо зашифровать.
+
+В отличии от аттрибутов `from_zk`, `from_env` и `incl` (или элемента `include`), подстановка, т.е. расшифровка зашифрованного значения, не выподняется в файле предобработки. Расшифровка происходит только во время исполнения в серверном процессе.
+
+Пример:
+
+```xml
+
+
+
+ 00112233445566778899aabbccddeeff
+
+
+
+ admin
+ 961F000000040000000000EEDDEF4F453CFE6457C4234BD7C09258BD651D85
+
+
+```
+
+Чтобы получить зашифрованное значение может быть использовано приложение-пример `encrypt_decrypt` .
+
+Пример:
+
+``` bash
+./encrypt_decrypt /etc/clickhouse-server/config.xml -e AES_128_GCM_SIV abcd
+```
+
+``` text
+961F000000040000000000EEDDEF4F453CFE6457C4234BD7C09258BD651D85
+```
+
## Примеры записи конфигурации на YAML {#example}
Здесь можно рассмотреть пример реальной конфигурации записанной на YAML: [config.yaml.example](https://github.com/ClickHouse/ClickHouse/blob/master/programs/server/config.yaml.example).
diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md
index 787153d4d19..421df3fe3eb 100644
--- a/docs/ru/operations/server-configuration-parameters/settings.md
+++ b/docs/ru/operations/server-configuration-parameters/settings.md
@@ -575,14 +575,60 @@ ClickHouse поддерживает динамическое изменение
- `errorlog` - Файл лога ошибок.
- `size` - Размер файла. Действует для `log` и `errorlog`. Как только файл достиг размера `size`, ClickHouse архивирует и переименовывает его, а на его месте создает новый файл лога.
- `count` - Количество заархивированных файлов логов, которые сохраняет ClickHouse.
+- `stream_compress` – Сжимать `log` и `errorlog` с помощью алгоритма `lz4`. Чтобы активировать, узтановите значение `1` или `true`.
+
+Имена файлов `log` и `errorlog` (только имя файла, а не директорий) поддерживают спецификаторы шаблонов даты и времени.
+
+**Спецификаторы форматирования**
+С помощью следующих спецификаторов, можно определить шаблон для формирования имени файла. Столбец “Пример” показывает возможные значения на момент времени `2023-07-06 18:32:07`.
+
+| Спецификатор | Описание | Пример |
+|--------------|---------------------------------------------------------------------------------------------------------------------|--------------------------|
+| %% | Литерал % | % |
+| %n | Символ новой строки | |
+| %t | Символ горизонтальной табуляции | |
+| %Y | Год как десятичное число, например, 2017 | 2023 |
+| %y | Последние 2 цифры года в виде десятичного числа (диапазон [00,99]) | 23 |
+| %C | Первые 2 цифры года в виде десятичного числа (диапазон [00,99]) | 20 |
+| %G | Год по неделям согласно [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Week_dates), то есть год, который содержит указанную неделю. Обычно используется вместе с %V. | 2023 |
+| %g | Последние 2 цифры [года по неделям ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Week_dates), т.е. года, содержащего указанную неделю (диапазон [00,99]). | 23 |
+| %b | Сокращённое название месяца, например Oct (зависит от локали) | Jul |
+| %h | Синоним %b | Jul |
+| %B | Полное название месяца, например, October (зависит от локали) | July |
+| %m | Месяц в виде десятичного числа (диапазон [01,12]) | 07 |
+| %U | Неделя года в виде десятичного числа (воскресенье - первый день недели) (диапазон [00,53]) | 27 |
+| %W | Неделя года в виде десятичного числа (понедельник - первый день недели) (диапазон [00,53]) | 27 |
+| %V | Неделя года ISO 8601 (диапазон [01,53]) | 27 |
+| %j | День года в виде десятичного числа (диапазон [001,366]) | 187 |
+| %d | День месяца в виде десятичного числа (диапазон [01,31]) Перед одиночной цифрой ставится ноль. | 06 |
+| %e | День месяца в виде десятичного числа (диапазон [1,31]). Перед одиночной цифрой ставится пробел. | 6 |
+| %a | Сокращённое название дня недели, например, Fri (зависит от локали) | Thu |
+| %A | Полный день недели, например, Friday (зависит от локали) | Thursday |
+| %w | День недели в виде десятичного числа, где воскресенье равно 0 (диапазон [0-6]) | 4 |
+| %u | День недели в виде десятичного числа, где понедельник равен 1 (формат ISO 8601) (диапазон [1-7]) | 4 |
+| %H | Час в виде десятичного числа, 24-часовой формат (диапазон [00-23]) | 18 |
+| %I | Час в виде десятичного числа, 12-часовой формат (диапазон [01,12]) | 06 |
+| %M | Минуты в виде десятичного числа (диапазон [00,59]) | 32 |
+| %S | Секунды как десятичное число (диапазон [00,60]) | 07 |
+| %c | Стандартная строка даты и времени, например, Sun Oct 17 04:41:13 2010 (зависит от локали) | Thu Jul 6 18:32:07 2023 |
+| %x | Локализованное представление даты (зависит от локали) | 07/06/23 |
+| %X | Локализованное представление времени, например, 18:40:20 или 6:40:20 PM (зависит от локали) | 18:32:07 |
+| %D | Эквивалентно "%m/%d/%y" | 07/06/23 |
+| %F | Эквивалентно "%Y-%m-%d" (формат даты ISO 8601) | 2023-07-06 |
+| %r | Локализованное 12-часовое время (зависит от локали) | 06:32:07 PM |
+| %R | Эквивалентно "%H:%M" | 18:32 |
+| %T | Эквивалентно "%H:%M:%S" (формат времени ISO 8601) | 18:32:07 |
+| %p | Локализованное обозначение a.m. или p.m. (зависит от локали) | PM |
+| %z | Смещение от UTC в формате ISO 8601 (например, -0430), или без символов, если информация о часовом поясе недоступна | +0800 |
+| %Z | Зависящее от локали название или аббревиатура часового пояса, если информация о часовом поясе доступна | Z AWST |
**Пример**
``` xml
trace
- /var/log/clickhouse-server/clickhouse-server.log
- /var/log/clickhouse-server/clickhouse-server.err.log
+ /var/log/clickhouse-server/clickhouse-server-%F-%T.log
+ /var/log/clickhouse-server/clickhouse-server-%F-%T.err.log
1000M
10
@@ -1067,7 +1113,7 @@ ClickHouse использует потоки из глобального пул
- requireTLSv1_2 - Требование соединения TLSv1.2. Допустимые значения: `true`, `false`.
- fips - Активация режима OpenSSL FIPS. Поддерживается, если версия OpenSSL, с которой собрана библиотека поддерживает fips.
- privateKeyPassphraseHandler - Класс (подкласс PrivateKeyPassphraseHandler)запрашивающий кодовую фразу доступа к секретному ключу. Например, ``, `KeyFileHandler`, `test`, ``.
-- invalidCertificateHandler - Класс (подкласс CertificateHandler) для подтверждения не валидных сертификатов. Например, ` ConsoleCertificateHandler `.
+- invalidCertificateHandler - Класс (подкласс CertificateHandler) для подтверждения не валидных сертификатов. Например, ` RejectCertificateHandler `.
- disableProtocols - Запрещенные к использованию протоколы.
- preferServerCiphers - Предпочтение серверных шифров на клиенте.
@@ -1355,6 +1401,10 @@ Parameters:
Europe/Moscow
```
+**См. также**
+
+- [session_timezone](../settings/settings.md#session_timezone)
+
## tcp_port {#server_configuration_parameters-tcp_port}
Порт для взаимодействия с клиентами по протоколу TCP.
diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md
index e3da8302fc8..957a917c780 100644
--- a/docs/ru/operations/settings/settings.md
+++ b/docs/ru/operations/settings/settings.md
@@ -1686,7 +1686,7 @@ SELECT * FROM table_with_enum_column_for_csv_insert;
## input_format_csv_detect_header {#input_format_csv_detect_header}
Обнаружить заголовок с именами и типами в формате CSV.
-
+
Значение по умолчанию - `true`.
## input_format_csv_skip_first_lines {#input_format_csv_skip_first_lines}
@@ -1727,6 +1727,12 @@ echo ' string ' | ./clickhouse local -q "select * from table FORMAT CSV" --in
" string "
```
+## input_format_csv_allow_variable_number_of_columns {#input_format_csv_allow_variable_number_of_columns}
+
+Игнорировать дополнительные столбцы (если файл содержит больше столбцов чем ожидается) и рассматривать отсутствующие поля в CSV в качестве значений по умолчанию.
+
+Выключено по умолчанию.
+
## output_format_tsv_crlf_end_of_line {#settings-output-format-tsv-crlf-end-of-line}
Использовать в качестве разделителя строк для TSV формата CRLF (DOC/Windows стиль) вместо LF (Unix стиль).
@@ -4127,6 +4133,63 @@ SELECT sum(number) FROM numbers(10000000000) SETTINGS partial_result_on_first_ca
Значение по умолчанию: `false`
+## session_timezone {#session_timezone}
+
+Задаёт значение часового пояса (session_timezone) по умолчанию для текущей сессии вместо [часового пояса сервера](../server-configuration-parameters/settings.md#server_configuration_parameters-timezone). То есть, все значения DateTime/DateTime64, для которых явно не задан часовой пояс, будут интерпретированы как относящиеся к указанной зоне.
+При значении настройки `''` (пустая строка), будет совпадать с часовым поясом сервера.
+
+Функции `timeZone()` and `serverTimezone()` возвращают часовой пояс текущей сессии и сервера соответственно.
+
+Примеры:
+```sql
+SELECT timeZone(), serverTimezone() FORMAT TSV
+
+Europe/Berlin Europe/Berlin
+```
+
+```sql
+SELECT timeZone(), serverTimezone() SETTINGS session_timezone = 'Asia/Novosibirsk' FORMAT TSV
+
+Asia/Novosibirsk Europe/Berlin
+```
+
+```sql
+SELECT toDateTime64(toDateTime64('1999-12-12 23:23:23.123', 3), 3, 'Europe/Zurich') SETTINGS session_timezone = 'America/Denver' FORMAT TSV
+
+1999-12-13 07:23:23.123
+```
+
+Возможные значения:
+
+- Любая зона из `system.time_zones`, например `Europe/Berlin`, `UTC` или `Zulu`
+
+Значение по умолчанию: `''`.
+
+:::warning
+Иногда при формировании значений типа `DateTime` и `DateTime64` параметр `session_timezone` может быть проигнорирован.
+Это может привести к путанице. Пример и пояснение см. ниже.
+:::
+
+```sql
+CREATE TABLE test_tz (`d` DateTime('UTC')) ENGINE = Memory AS SELECT toDateTime('2000-01-01 00:00:00', 'UTC');
+
+SELECT *, timezone() FROM test_tz WHERE d = toDateTime('2000-01-01 00:00:00') SETTINGS session_timezone = 'Asia/Novosibirsk'
+0 rows in set.
+
+SELECT *, timezone() FROM test_tz WHERE d = '2000-01-01 00:00:00' SETTINGS session_timezone = 'Asia/Novosibirsk'
+┌───────────────────d─┬─timezone()───────┐
+│ 2000-01-01 00:00:00 │ Asia/Novosibirsk │
+└─────────────────────┴──────────────────┘
+```
+
+Это происходит из-за различного происхождения значения, используемого для сравнения:
+- В первом запросе функция `toDateTime()`, создавая значение типа `DateTime`, принимает во внимание параметр `session_timezone` из контекста запроса;
+- Во втором запросе `DateTime` формируется из строки неявно, наследуя тип колонки `d` (в том числе и числовой пояс), и параметр `session_timezone` игнорируется.
+
+**Смотрите также**
+
+- [timezone](../server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
+
## rename_files_after_processing
- **Тип:** Строка
@@ -4138,6 +4201,7 @@ SELECT sum(number) FROM numbers(10000000000) SETTINGS partial_result_on_first_ca
### Шаблон
Шаблон поддерживает следующие виды плейсхолдеров:
+- `%a` — Полное исходное имя файла (например "sample.csv").
- `%f` — Исходное имя файла без расширения (например "sample").
- `%e` — Оригинальное расширение файла с точкой (например ".csv").
- `%t` — Текущее время (в микросекундах).
diff --git a/docs/ru/operations/system-tables/asynchronous_metric_log.md b/docs/ru/operations/system-tables/asynchronous_metric_log.md
index 886fbb6cab0..5145889c95f 100644
--- a/docs/ru/operations/system-tables/asynchronous_metric_log.md
+++ b/docs/ru/operations/system-tables/asynchronous_metric_log.md
@@ -8,7 +8,6 @@ slug: /ru/operations/system-tables/asynchronous_metric_log
Столбцы:
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата события.
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время события.
-- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время события в микросекундах.
- `name` ([String](../../sql-reference/data-types/string.md)) — название метрики.
- `value` ([Float64](../../sql-reference/data-types/float.md)) — значение метрики.
diff --git a/docs/ru/operations/system-tables/query_log.md b/docs/ru/operations/system-tables/query_log.md
index a55528bd829..8f858c14fb1 100644
--- a/docs/ru/operations/system-tables/query_log.md
+++ b/docs/ru/operations/system-tables/query_log.md
@@ -69,11 +69,11 @@ ClickHouse не удаляет данные из таблица автомати
- 0 — запрос был инициирован другим запросом при выполнении распределенного запроса.
- `user` ([String](../../sql-reference/data-types/string.md)) — пользователь, запустивший текущий запрос.
- `query_id` ([String](../../sql-reference/data-types/string.md)) — ID запроса.
-- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел запрос.
+- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP адрес, с которого пришел запрос.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт, с которого клиент сделал запрос
- `initial_user` ([String](../../sql-reference/data-types/string.md)) — пользователь, запустивший первоначальный запрос (для распределенных запросов).
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID родительского запроса.
-- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
+- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт, с которого клиент сделал родительский запрос.
- `initial_query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса (для распределенных запросов).
- `initial_query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд (для распределенных запросов).
diff --git a/docs/ru/operations/system-tables/query_thread_log.md b/docs/ru/operations/system-tables/query_thread_log.md
index c9aabb02cad..1a256e1657a 100644
--- a/docs/ru/operations/system-tables/query_thread_log.md
+++ b/docs/ru/operations/system-tables/query_thread_log.md
@@ -39,11 +39,11 @@ ClickHouse не удаляет данные из таблицы автомати
- 0 — запрос был инициирован другим запросом при распределенном запросе.
- `user` ([String](../../sql-reference/data-types/string.md)) — пользователь, запустивший текущий запрос.
- `query_id` ([String](../../sql-reference/data-types/string.md)) — ID запроса.
-- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел запрос.
+- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP адрес, с которого пришел запрос.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — порт, с которого пришел запрос.
- `initial_user` ([String](../../sql-reference/data-types/string.md)) — пользователь, запустивший первоначальный запрос (для распределенных запросов).
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID родительского запроса.
-- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
+- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — порт, пришел родительский запрос.
- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — интерфейс, с которого ушёл запрос. Возможные значения:
- 1 — TCP.
diff --git a/docs/ru/operations/system-tables/session_log.md b/docs/ru/operations/system-tables/session_log.md
index 1f313e7815a..5849cb51ab4 100644
--- a/docs/ru/operations/system-tables/session_log.md
+++ b/docs/ru/operations/system-tables/session_log.md
@@ -27,7 +27,7 @@ slug: /ru/operations/system-tables/session_log
- `profiles` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — список профилей, установленных для всех ролей и (или) пользователей.
- `roles` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — список ролей, к которым применяется данный профиль.
- `settings` ([Array](../../sql-reference/data-types/array.md)([Tuple](../../sql-reference/data-types/tuple.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md), [String](../../sql-reference/data-types/string.md)))) — настройки, которые были изменены при входе или выходе клиента из системы.
-- `client_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP-адрес, который использовался для входа или выхода из системы.
+- `client_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP-адрес, который использовался для входа или выхода из системы.
- `client_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт клиента, который использовался для входа или выхода из системы.
- `interface` ([Enum8](../../sql-reference/data-types/enum.md)) — интерфейс, с которого был инициирован вход в систему. Возможные значения:
- `TCP`
diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md
index ccbdd5110ad..9874cb3a269 100644
--- a/docs/ru/operations/system-tables/zookeeper_log.md
+++ b/docs/ru/operations/system-tables/zookeeper_log.md
@@ -15,7 +15,7 @@ slug: /ru/operations/system-tables/zookeeper_log
- `Finalize` — соединение разорвано, ответ не получен.
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата, когда произошло событие.
- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — дата и время, когда произошло событие.
-- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес сервера ZooKeeper, с которого был сделан запрос.
+- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP адрес сервера ZooKeeper, с которого был сделан запрос.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт сервера ZooKeeper, с которого был сделан запрос.
- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор сессии, который сервер ZooKeeper создает для каждого соединения.
- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — идентификатор запроса внутри сессии. Обычно это последовательный номер запроса, одинаковый у строки запроса и у парной строки `response`/`finalize`.
diff --git a/docs/ru/sql-reference/aggregate-functions/combinators.md b/docs/ru/sql-reference/aggregate-functions/combinators.md
index 3a7ff571f99..99d5f11442c 100644
--- a/docs/ru/sql-reference/aggregate-functions/combinators.md
+++ b/docs/ru/sql-reference/aggregate-functions/combinators.md
@@ -66,6 +66,10 @@ WITH anySimpleState(number) AS c SELECT toTypeName(c), c FROM numbers(1);
В случае применения этого комбинатора, агрегатная функция возвращает не готовое значение (например, в случае функции [uniq](reference/uniq.md#agg_function-uniq) — количество уникальных значений), а промежуточное состояние агрегации (например, в случае функции `uniq` — хэш-таблицу для расчёта количества уникальных значений), которое имеет тип `AggregateFunction(...)` и может использоваться для дальнейшей обработки или может быть сохранено в таблицу для последующей доагрегации.
+:::note
+Промежуточное состояние для -MapState не является инвариантом для одних и тех же исходных данные т.к. порядок данных может меняться. Это не влияет, тем не менее, на загрузку таких данных.
+:::
+
Для работы с промежуточными состояниями предназначены:
- Движок таблиц [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md).
diff --git a/docs/ru/sql-reference/data-types/datetime.md b/docs/ru/sql-reference/data-types/datetime.md
index e8d4a3ee9fd..80d844a1713 100644
--- a/docs/ru/sql-reference/data-types/datetime.md
+++ b/docs/ru/sql-reference/data-types/datetime.md
@@ -122,6 +122,7 @@ FROM dt
- [Настройка `date_time_input_format`](../../operations/settings/index.md#settings-date_time_input_format)
- [Настройка `date_time_output_format`](../../operations/settings/index.md)
- [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
+- [Параметр `session_timezone`](../../operations/settings/settings.md#session_timezone)
- [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime)
- [Тип данных `Date`](date.md)
- [Тип данных `DateTime64`](datetime64.md)
diff --git a/docs/ru/sql-reference/data-types/datetime64.md b/docs/ru/sql-reference/data-types/datetime64.md
index da2f81f4828..78ad43e4764 100644
--- a/docs/ru/sql-reference/data-types/datetime64.md
+++ b/docs/ru/sql-reference/data-types/datetime64.md
@@ -102,6 +102,7 @@ FROM dt;
- [Настройка `date_time_input_format`](../../operations/settings/settings.md#settings-date_time_input_format)
- [Настройка `date_time_output_format`](../../operations/settings/settings.md)
- [Конфигурационный параметр сервера `timezone`](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
+- [Параметр `session_timezone`](../../operations/settings/settings.md#session_timezone)
- [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime)
- [Тип данных `Date`](date.md)
- [Тип данных `DateTime`](datetime.md)
diff --git a/docs/ru/sql-reference/data-types/decimal.md b/docs/ru/sql-reference/data-types/decimal.md
index 81cb5079945..dbbf18253b2 100644
--- a/docs/ru/sql-reference/data-types/decimal.md
+++ b/docs/ru/sql-reference/data-types/decimal.md
@@ -31,7 +31,7 @@ sidebar_label: Decimal
## Внутреннее представление {#vnutrennee-predstavlenie}
Внутри данные представляются как знаковые целые числа, соответсвующей разрядности. Реальные диапазоны, хранящиеся в ячейках памяти несколько больше заявленных. Заявленные диапазоны Decimal проверяются только при вводе числа из строкового представления.
-Поскольку современные CPU не поддерживают 128-битные числа, операции над Decimal128 эмулируются программно. Decimal128 работает в разы медленней чем Decimal32/Decimal64.
+Поскольку современные CPU не поддерживают 128-битные и 256-битные числа, для операций над Decimal128 и Decimal256 эмулируются программно. Данные типы работают в разы медленнее, чем Decimal32/Decimal64.
## Операции и типы результата {#operatsii-i-tipy-rezultata}
@@ -59,6 +59,10 @@ sidebar_label: Decimal
При выполнении операций над типом Decimal могут происходить целочисленные переполнения. Лишняя дробная часть отбрасывается (не округляется). Лишняя целочисленная часть приводит к исключению.
+:::warning
+Проверка переполнения не реализована для Decimal128 и Decimal256. В случае переполнения неверный результат будёт возвращён без выбрасывания исключения.
+:::
+
``` sql
SELECT toDecimal32(2, 4) AS x, x / 3
```
diff --git a/docs/ru/sql-reference/data-types/domains/ipv4.md b/docs/ru/sql-reference/data-types/ipv4.md
similarity index 56%
rename from docs/ru/sql-reference/data-types/domains/ipv4.md
rename to docs/ru/sql-reference/data-types/ipv4.md
index 57a19e282ae..5cb977c64c9 100644
--- a/docs/ru/sql-reference/data-types/domains/ipv4.md
+++ b/docs/ru/sql-reference/data-types/ipv4.md
@@ -1,12 +1,12 @@
---
-slug: /ru/sql-reference/data-types/domains/ipv4
+slug: /ru/sql-reference/data-types/ipv4
sidebar_position: 59
sidebar_label: IPv4
---
## IPv4 {#ipv4}
-`IPv4` — это домен, базирующийся на типе данных `UInt32` предназначенный для хранения адресов IPv4. Он обеспечивает компактное хранение данных с удобным для человека форматом ввода-вывода, и явно отображаемым типом данных в структуре таблицы.
+IPv4-адреса. Хранится в 4 байтах как UInt32.
### Применение {#primenenie}
@@ -57,27 +57,6 @@ SELECT toTypeName(from), hex(from) FROM hits LIMIT 1;
└──────────────────┴───────────┘
```
-Значения с доменным типом данных не преобразуются неявно в другие типы данных, кроме `UInt32`.
-Если необходимо преобразовать значение типа `IPv4` в строку, то это необходимо делать явно с помощью функции `IPv4NumToString()`:
+**См. также**
-``` sql
-SELECT toTypeName(s), IPv4NumToString(from) AS s FROM hits LIMIT 1;
-```
-
-``` text
-┌─toTypeName(IPv4NumToString(from))─┬─s──────────────┐
-│ String │ 183.247.232.58 │
-└───────────────────────────────────┴────────────────┘
-```
-
-Или приводить к типу данных `UInt32`:
-
-``` sql
-SELECT toTypeName(i), CAST(from AS UInt32) AS i FROM hits LIMIT 1;
-```
-
-``` text
-┌─toTypeName(CAST(from, 'UInt32'))─┬──────────i─┐
-│ UInt32 │ 3086477370 │
-└──────────────────────────────────┴────────────┘
-```
+- [Functions for Working with IPv4 and IPv6 Addresses](../functions/ip-address-functions.md)
diff --git a/docs/ru/sql-reference/data-types/domains/ipv6.md b/docs/ru/sql-reference/data-types/ipv6.md
similarity index 98%
rename from docs/ru/sql-reference/data-types/domains/ipv6.md
rename to docs/ru/sql-reference/data-types/ipv6.md
index fdfb26f68c1..808068ce90a 100644
--- a/docs/ru/sql-reference/data-types/domains/ipv6.md
+++ b/docs/ru/sql-reference/data-types/ipv6.md
@@ -1,5 +1,5 @@
---
-slug: /ru/sql-reference/data-types/domains/ipv6
+slug: /ru/sql-reference/data-types/ipv6
sidebar_position: 60
sidebar_label: IPv6
---
diff --git a/docs/ru/sql-reference/functions/array-functions.md b/docs/ru/sql-reference/functions/array-functions.md
index c43323d68fd..439eddfd752 100644
--- a/docs/ru/sql-reference/functions/array-functions.md
+++ b/docs/ru/sql-reference/functions/array-functions.md
@@ -145,6 +145,8 @@ range([start, ] end [, step])
- Если в результате запроса создаются массивы суммарной длиной больше, чем количество элементов, указанное настройкой [function_range_max_elements_in_block](../../operations/settings/settings.md#settings-function_range_max_elements_in_block), то генерируется исключение.
+- Возвращает Null если любой аргумент Nullable(Nothing) типа. Генерируется исключение если любой аргумент Null (Nullable(T) тип).
+
**Примеры**
Запрос:
diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md
index 867d71d334c..4db8a1ec6f8 100644
--- a/docs/ru/sql-reference/functions/date-time-functions.md
+++ b/docs/ru/sql-reference/functions/date-time-functions.md
@@ -26,7 +26,8 @@ SELECT
## timeZone {#timezone}
-Возвращает часовой пояс сервера.
+Возвращает часовой пояс сервера, считающийся умолчанием для текущей сессии: значение параметра [session_timezone](../../operations/settings/settings.md#session_timezone), если установлено.
+
Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями, актуальными для каждого шарда. Иначе возвращается константа.
**Синтаксис**
@@ -43,6 +44,33 @@ timeZone()
Тип: [String](../../sql-reference/data-types/string.md).
+**Смотрите также**
+
+- [serverTimeZone](#servertimezone)
+
+## serverTimeZone {#servertimezone}
+
+Возвращает часовой пояс сервера по умолчанию, в т.ч. установленный [timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
+Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями, актуальными для каждого шарда. Иначе возвращается константа.
+
+**Синтаксис**
+
+``` sql
+serverTimeZone()
+```
+
+Синонимы: `serverTimezone`.
+
+**Возвращаемое значение**
+
+- Часовой пояс.
+
+Тип: [String](../../sql-reference/data-types/string.md).
+
+**Смотрите также**
+
+- [timeZone](#timezone)
+
## toTimeZone {#totimezone}
Переводит дату или дату с временем в указанный часовой пояс. Часовой пояс - это атрибут типов `Date` и `DateTime`. Внутреннее значение (количество секунд) поля таблицы или результирующего столбца не изменяется, изменяется тип поля и, соответственно, его текстовое отображение.
@@ -571,29 +599,33 @@ SELECT toDate('2016-12-27') AS date, toWeek(date) AS week0, toWeek(date,1) AS we
## toYearWeek(date[,mode]) {#toyearweek}
Возвращает год и неделю для даты. Год в результате может отличаться от года в аргументе даты для первой и последней недели года.
-Аргумент mode работает точно так же, как аргумент mode [toWeek()](#toweek). Если mode не задан, используется режим 0.
+Аргумент mode работает так же, как аргумент mode [toWeek()](#toweek), значение mode по умолчанию -- `0`.
-`toISOYear() ` эквивалентно `intDiv(toYearWeek(date,3),100)`.
+`toISOYear() ` эквивалентно `intDiv(toYearWeek(date,3),100)`
+
+:::warning
+Однако, есть отличие в работе функций `toWeek()` и `toYearWeek()`. `toWeek()` возвращает номер недели в контексте заданного года, и в случае, когда `toWeek()` вернёт `0`, `toYearWeek()` вернёт значение, соответствующее последней неделе предыдущего года (см. `prev_yearWeek` в примере).
+:::
**Пример**
Запрос:
```sql
-SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(date,1) AS yearWeek1, toYearWeek(date,9) AS yearWeek9;
+SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(date,1) AS yearWeek1, toYearWeek(date,9) AS yearWeek9, toYearWeek(toDate('2022-01-01')) AS prev_yearWeek;
```
Результат:
```text
-┌───────date─┬─yearWeek0─┬─yearWeek1─┬─yearWeek9─┐
-│ 2016-12-27 │ 201652 │ 201652 │ 201701 │
-└────────────┴───────────┴───────────┴───────────┘
+┌───────date─┬─yearWeek0─┬─yearWeek1─┬─yearWeek9─┬─prev_yearWeek─┐
+│ 2016-12-27 │ 201652 │ 201652 │ 201701 │ 202152 │
+└────────────┴───────────┴───────────┴───────────┴───────────────┘
```
## age
-Вычисляет компонент `unit` разницы между `startdate` и `enddate`. Разница вычисляется с точностью в 1 секунду.
+Вычисляет компонент `unit` разницы между `startdate` и `enddate`. Разница вычисляется с точностью в 1 микросекунду.
Например, разница между `2021-12-29` и `2022-01-01` 3 дня для единицы `day`, 0 месяцев для единицы `month`, 0 лет для единицы `year`.
**Синтаксис**
@@ -607,6 +639,8 @@ age('unit', startdate, enddate, [timezone])
- `unit` — единица измерения времени, в которой будет выражено возвращаемое значение функции. [String](../../sql-reference/data-types/string.md).
Возможные значения:
+ - `microsecond` (возможные сокращения: `us`, `u`)
+ - `millisecond` (возможные сокращения: `ms`)
- `second` (возможные сокращения: `ss`, `s`)
- `minute` (возможные сокращения: `mi`, `n`)
- `hour` (возможные сокращения: `hh`, `h`)
@@ -680,6 +714,8 @@ date_diff('unit', startdate, enddate, [timezone])
- `unit` — единица измерения времени, в которой будет выражено возвращаемое значение функции. [String](../../sql-reference/data-types/string.md).
Возможные значения:
+ - `microsecond` (возможные сокращения: `us`, `u`)
+ - `millisecond` (возможные сокращения: `ms`)
- `second` (возможные сокращения: `ss`, `s`)
- `minute` (возможные сокращения: `mi`, `n`)
- `hour` (возможные сокращения: `hh`, `h`)
diff --git a/docs/ru/sql-reference/functions/ext-dict-functions.md b/docs/ru/sql-reference/functions/ext-dict-functions.md
index e6cb878d1c7..d14f0ddf027 100644
--- a/docs/ru/sql-reference/functions/ext-dict-functions.md
+++ b/docs/ru/sql-reference/functions/ext-dict-functions.md
@@ -5,7 +5,7 @@ sidebar_label: "Функции для работы с внешними слов
---
:::note "Внимание"
- Для словарей, созданных с помощью [DDL-запросов](../../sql-reference/statements/create/dictionary.md), в параметре `dict_name` указывается полное имя словаря вместе с базой данных, например: `.`. Если база данных не указана, используется текущая.
+Для словарей, созданных с помощью [DDL-запросов](../../sql-reference/statements/create/dictionary.md), в параметре `dict_name` указывается полное имя словаря вместе с базой данных, например: `.`. Если база данных не указана, используется текущая.
:::
# Функции для работы с внешними словарями {#ext_dict_functions}
diff --git a/docs/ru/sql-reference/functions/ip-address-functions.md b/docs/ru/sql-reference/functions/ip-address-functions.md
index 96d4b737c88..d1a72b82b67 100644
--- a/docs/ru/sql-reference/functions/ip-address-functions.md
+++ b/docs/ru/sql-reference/functions/ip-address-functions.md
@@ -265,7 +265,7 @@ SELECT
## toIPv6 {#toipv6string}
-Приводит строку с адресом в формате IPv6 к типу [IPv6](../../sql-reference/data-types/domains/ipv6.md). Возвращает пустое значение, если входящая строка не является корректным IP адресом.
+Приводит строку с адресом в формате IPv6 к типу [IPv6](../../sql-reference/data-types/ipv6.md). Возвращает пустое значение, если входящая строка не является корректным IP адресом.
Похоже на функцию [IPv6StringToNum](#ipv6stringtonums), которая представляет адрес IPv6 в двоичном виде.
Если входящая строка содержит корректный IPv4 адрес, функция возвращает его IPv6 эквивалент.
@@ -284,7 +284,7 @@ toIPv6(string)
- IP адрес.
-Тип: [IPv6](../../sql-reference/data-types/domains/ipv6.md).
+Тип: [IPv6](../../sql-reference/data-types/ipv6.md).
**Примеры**
diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md
index 9638e25d488..276dfc2ef20 100644
--- a/docs/ru/sql-reference/functions/string-functions.md
+++ b/docs/ru/sql-reference/functions/string-functions.md
@@ -1113,3 +1113,50 @@ A text with tags .
The content within CDATA
Do Nothing for 2 Minutes 2:00
```
+
+## initcap {#initcap}
+
+Переводит первую букву каждого слова в строке в верхний регистр, а остальные — в нижний. Словами считаются последовательности алфавитно-цифровых символов, разделённые любыми другими символами.
+
+## initcapUTF8 {#initcapUTF8}
+
+Как [initcap](#initcap), предполагая, что строка содержит набор байтов, представляющий текст в кодировке UTF-8.
+Не учитывает язык. То есть, для турецкого языка, результат может быть не совсем верным.
+Если длина UTF-8 последовательности байтов различна для верхнего и нижнего регистра кодовой точки, то для этой кодовой точки результат работы может быть некорректным.
+Если строка содержит набор байтов, не являющийся UTF-8, то поведение не определено.
+
+## firstLine
+
+Возвращает первую строку в многострочном тексте.
+
+**Синтаксис**
+
+```sql
+firstLine(val)
+```
+
+**Аргументы**
+
+- `val` - текст для обработки. [String](../data-types/string.md)
+
+**Returned value**
+
+- Первая строка текста или весь текст, если переносы строк отсутствуют.
+
+Тип: [String](../data-types/string.md)
+
+**Пример**
+
+Запрос:
+
+```sql
+select firstLine('foo\nbar\nbaz');
+```
+
+Результат:
+
+```result
+┌─firstLine('foo\nbar\nbaz')─┐
+│ foo │
+└────────────────────────────┘
+```
diff --git a/docs/ru/sql-reference/functions/string-search-functions.md b/docs/ru/sql-reference/functions/string-search-functions.md
index ea4f90d4f66..6e3830869cd 100644
--- a/docs/ru/sql-reference/functions/string-search-functions.md
+++ b/docs/ru/sql-reference/functions/string-search-functions.md
@@ -801,3 +801,55 @@ SELECT countSubstringsCaseInsensitiveUTF8('аБв__АбВ__абв', 'Абв');
│ 3 │
└────────────────────────────────────────────────────────────┘
```
+
+## hasSubsequence(haystack, needle) {#hasSubsequence}
+
+Возвращает 1 если needle является подпоследовательностью haystack, иначе 0.
+
+
+**Синтаксис**
+
+``` sql
+hasSubsequence(haystack, needle)
+```
+
+**Аргументы**
+
+- `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
+- `needle` — подпоследовательность, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal).
+
+**Возвращаемые значения**
+
+- 1, если
+- 0, если подстрока не найдена.
+
+Тип: `UInt8`.
+
+**Примеры**
+
+Запрос:
+
+``` sql
+SELECT hasSubsequence('garbage', 'arg') ;
+```
+
+Результат:
+
+``` text
+┌─hasSubsequence('garbage', 'arg')─┐
+│ 1 │
+└──────────────────────────────────┘
+```
+
+
+## hasSubsequenceCaseInsensitive
+
+Такая же, как и [hasSubsequence](#hasSubsequence), но работает без учета регистра.
+
+## hasSubsequenceUTF8
+
+Такая же, как и [hasSubsequence](#hasSubsequence) при допущении что `haystack` и `needle` содержат набор кодовых точек, представляющий текст в кодировке UTF-8.
+
+## hasSubsequenceCaseInsensitiveUTF8
+
+Такая же, как и [hasSubsequenceUTF8](#hasSubsequenceUTF8), но работает без учета регистра.
diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md
index d5e6246fe9e..e53104d8d71 100644
--- a/docs/ru/sql-reference/functions/type-conversion-functions.md
+++ b/docs/ru/sql-reference/functions/type-conversion-functions.md
@@ -284,7 +284,13 @@ toDateTime(expr[, time_zone ])
- `expr` — Значение для преобразования. [String](/docs/ru/sql-reference/data-types/string.md), [Int](/docs/ru/sql-reference/data-types/int-uint.md), [Date](/docs/ru/sql-reference/data-types/date.md) или [DateTime](/docs/ru/sql-reference/data-types/datetime.md).
- `time_zone` — Часовой пояс. [String](/docs/ru/sql-reference/data-types/string.md).
-Если `expr` является числом, оно интерпретируется как количество секунд от начала unix эпохи.
+:::note
+Если `expr` является числом, то оно интерпретируется как число секунд с начала Unix-эпохи (Unix Timestamp).
+
+Если же `expr` -- [строка (String)](/docs/ru/sql-reference/data-types/string.md), то оно может быть интерпретировано и как Unix Timestamp, и как строковое представление даты / даты со временем.
+Ввиду неоднозначности запрещён парсинг строк длиной 4 и меньше. Так, строка `'1999'` могла бы представлять собой как год (неполное строковое представление даты или даты со временем), так и Unix Timestamp.
+Строки длиной 5 символов и более не несут неоднозначности, а следовательно, их парсинг разрешён.
+:::
**Возвращаемое значение**
diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md
index a8ace213075..92be30b101a 100644
--- a/docs/ru/sql-reference/statements/alter/column.md
+++ b/docs/ru/sql-reference/statements/alter/column.md
@@ -182,7 +182,7 @@ ALTER TABLE visits MODIFY COLUMN browser Array(String)
Синтаксис:
```sql
-ALTER TABLE table_name MODIFY column_name REMOVE property;
+ALTER TABLE table_name MODIFY COLUMN column_name REMOVE property;
```
**Пример**
diff --git a/docs/ru/sql-reference/statements/create/view.md b/docs/ru/sql-reference/statements/create/view.md
index d3846aac289..1a60dc0716c 100644
--- a/docs/ru/sql-reference/statements/create/view.md
+++ b/docs/ru/sql-reference/statements/create/view.md
@@ -73,7 +73,7 @@ CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]na
Чтобы использовать `LIVE VIEW` и запросы `WATCH`, включите настройку [allow_experimental_live_view](../../../operations/settings/settings.md#allow-experimental-live-view).
:::
```sql
-CREATE LIVE VIEW [IF NOT EXISTS] [db.]table_name [WITH [TIMEOUT [value_in_sec] [AND]] [REFRESH [value_in_sec]]] AS SELECT ...
+CREATE LIVE VIEW [IF NOT EXISTS] [db.]table_name [WITH REFRESH [value_in_sec]] AS SELECT ...
```
`LIVE VIEW` хранит результат запроса [SELECT](../../../sql-reference/statements/select/index.md), указанного при создании, и обновляется сразу же при изменении этого результата. Конечный результат запроса и промежуточные данные, из которых формируется результат, хранятся в оперативной памяти, и это обеспечивает высокую скорость обработки для повторяющихся запросов. LIVE-представления могут отправлять push-уведомления при изменении результата исходного запроса `SELECT`. Для этого используйте запрос [WATCH](../../../sql-reference/statements/watch.md).
diff --git a/docs/ru/sql-reference/table-functions/file.md b/docs/ru/sql-reference/table-functions/file.md
index 0983c51d954..83ef115aacd 100644
--- a/docs/ru/sql-reference/table-functions/file.md
+++ b/docs/ru/sql-reference/table-functions/file.md
@@ -79,7 +79,7 @@ SELECT * FROM file('test.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 U
- `*` — заменяет любое количество любых символов кроме `/`, включая отсутствие символов.
- `?` — заменяет ровно один любой символ.
-- `{some_string,another_string,yet_another_one}` — заменяет любую из строк `'some_string', 'another_string', 'yet_another_one'`.
+- `{some_string,another_string,yet_another_one}` — заменяет любую из строк `'some_string', 'another_string', 'yet_another_one'`, причём строка может содержать `/`.
- `{N..M}` — заменяет любое число в интервале от `N` до `M` включительно (может содержать ведущие нули).
Конструкция с `{}` аналогична табличной функции [remote](remote.md).
diff --git a/docs/zh/development/build.md b/docs/zh/development/build.md
index d76f4b1577c..bb25755a615 100644
--- a/docs/zh/development/build.md
+++ b/docs/zh/development/build.md
@@ -3,13 +3,6 @@ slug: /zh/development/build
---
# 如何构建 ClickHouse 发布包 {#ru-he-gou-jian-clickhouse-fa-bu-bao}
-## 安装 Git 和 Pbuilder {#an-zhuang-git-he-pbuilder}
-
-``` bash
-sudo apt-get update
-sudo apt-get install git pbuilder debhelper lsb-release fakeroot sudo debian-archive-keyring debian-keyring
-```
-
## 拉取 ClickHouse 源码 {#la-qu-clickhouse-yuan-ma}
``` bash
diff --git a/docs/zh/interfaces/http.md b/docs/zh/interfaces/http.md
index c7a0f355a92..f84768beccc 100644
--- a/docs/zh/interfaces/http.md
+++ b/docs/zh/interfaces/http.md
@@ -53,7 +53,7 @@ Connection: Close
Content-Type: text/tab-separated-values; charset=UTF-8
X-ClickHouse-Server-Display-Name: clickhouse.ru-central1.internal
X-ClickHouse-Query-Id: 5abe861c-239c-467f-b955-8a201abb8b7f
-X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
1
```
@@ -262,9 +262,9 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812
您可以在`X-ClickHouse-Progress`响应头中收到查询进度的信息。为此，启用[Http Header携带进度](../operations/settings/settings.md#settings-send_progress_in_http_headers)。示例：
``` text
-X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128"}
-X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128"}
-X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128"}
+X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128","peak_memory_usage":"4371480"}
+X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128","peak_memory_usage":"13621616"}
+X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128","peak_memory_usage":"23155600"}
```
显示字段信息:
@@ -363,7 +363,7 @@ $ curl -v 'http://localhost:8123/predefined_query'
< X-ClickHouse-Format: Template
< X-ClickHouse-Timezone: Asia/Shanghai
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
# HELP "Query" "Number of executing queries"
# TYPE "Query" counter
@@ -521,7 +521,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
* Connection #0 to host localhost left intact
Say Hi!%
@@ -561,7 +561,7 @@ $ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler'
< Content-Type: text/plain; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
* Connection #0 to host localhost left intact
%
@@ -613,7 +613,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
Absolute Path File
* Connection #0 to host localhost left intact
@@ -632,7 +632,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","peak_memory_usage":"0"}
<
Relative Path File
* Connection #0 to host localhost left intact
diff --git a/docs/zh/operations/server-configuration-parameters/settings.md b/docs/zh/operations/server-configuration-parameters/settings.md
index 52142eda2e8..f6106d8734e 100644
--- a/docs/zh/operations/server-configuration-parameters/settings.md
+++ b/docs/zh/operations/server-configuration-parameters/settings.md
@@ -466,7 +466,7 @@ SSL客户端/服务器配置。
- requireTLSv1_2 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`.
- fips – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS.
- privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ``, `KeyFileHandler`, `test`, ``.
-- invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` ConsoleCertificateHandler ` .
+- invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` RejectCertificateHandler ` .
- disableProtocols – Protocols that are not allowed to use.
- preferServerCiphers – Preferred server ciphers on the client.
diff --git a/docs/zh/operations/system-tables/asynchronous_metric_log.md b/docs/zh/operations/system-tables/asynchronous_metric_log.md
index 419ad2a7ed6..9fa399f1aed 100644
--- a/docs/zh/operations/system-tables/asynchronous_metric_log.md
+++ b/docs/zh/operations/system-tables/asynchronous_metric_log.md
@@ -8,7 +8,6 @@ slug: /zh/operations/system-tables/asynchronous_metric_log
列：
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — 事件日期。
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — 事件时间。
-- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — 事件时间(微秒)。
- `name` ([String](../../sql-reference/data-types/string.md)) — 指标名。
- `value` ([Float64](../../sql-reference/data-types/float.md)) — 指标值。
@@ -17,18 +16,18 @@ slug: /zh/operations/system-tables/asynchronous_metric_log
SELECT * FROM system.asynchronous_metric_log LIMIT 10
```
``` text
-┌─event_date─┬──────────event_time─┬────event_time_microseconds─┬─name─────────────────────────────────────┬─────value─┐
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ CPUFrequencyMHz_0 │ 2120.9 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pmuzzy │ 743 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pdirty │ 26288 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.run_intervals │ 0 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.num_runs │ 0 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.retained │ 60694528 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.mapped │ 303161344 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.resident │ 260931584 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.metadata │ 12079488 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.allocated │ 133756128 │
-└────────────┴─────────────────────┴────────────────────────────┴──────────────────────────────────────────┴───────────┘
+┌─event_date─┬──────────event_time─┬─name─────────────────────────────────────┬─────value─┐
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ CPUFrequencyMHz_0 │ 2120.9 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.arenas.all.pmuzzy │ 743 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.arenas.all.pdirty │ 26288 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.background_thread.run_intervals │ 0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.background_thread.num_runs │ 0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.retained │ 60694528 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.mapped │ 303161344 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.resident │ 260931584 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.metadata │ 12079488 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.allocated │ 133756128 │
+└────────────┴─────────────────────┴──────────────────────────────────────────┴───────────┘
```
**另请参阅**
diff --git a/docs/zh/operations/system-tables/query_log.md b/docs/zh/operations/system-tables/query_log.md
index 7149282dfcc..0ba669906cb 100644
--- a/docs/zh/operations/system-tables/query_log.md
+++ b/docs/zh/operations/system-tables/query_log.md
@@ -60,11 +60,11 @@ ClickHouse不会自动从表中删除数据。更多详情请看 [introduction](
- 0 — 由另一个查询发起的，作为分布式查询的一部分.
- `user` ([String](../../sql-reference/data-types/string.md)) — 发起查询的用户.
- `query_id` ([String](../../sql-reference/data-types/string.md)) — 查询ID.
-- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起查询的客户端IP地址.
+- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — 发起查询的客户端IP地址.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — 发起查询的客户端端口.
- `initial_user` ([String](../../sql-reference/data-types/string.md)) — 初始查询的用户名（用于分布式查询执行）.
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — 运行初始查询的ID（用于分布式查询执行）.
-- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 运行父查询的IP地址.
+- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — 运行父查询的IP地址.
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — 发起父查询的客户端端口.
- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md)) — 发起查询的接口. 可能的值:
- 1 — TCP.
diff --git a/docs/zh/operations/system-tables/query_thread_log.md b/docs/zh/operations/system-tables/query_thread_log.md
index 8a41c1501a6..c4b7e2f1043 100644
--- a/docs/zh/operations/system-tables/query_thread_log.md
+++ b/docs/zh/operations/system-tables/query_thread_log.md
@@ -36,11 +36,11 @@ ClickHouse不会自动从表中删除数据。 欲了解更多详情，请参照
- 0 — 由其他查询发起的分布式查询。
- `user` ([字符串](../../sql-reference/data-types/string.md)) — 发起查询的用户名。
- `query_id` ([字符串](../../sql-reference/data-types/string.md)) — 查询的ID。
-- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起查询的IP地址。
+- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — 发起查询的IP地址。
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起查询的端口。
- `initial_user` ([字符串](../../sql-reference/data-types/string.md)) — 首次发起查询的用户名（对于分布式查询）。
- `initial_query_id` ([字符串](../../sql-reference/data-types/string.md)) — 首次发起查询的ID（对于分布式查询）。
-- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起该查询的父查询IP地址。
+- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — 发起该查询的父查询IP地址。
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起该查询的父查询端口。
- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起查询的界面，可能的值:
- 1 — TCP.
diff --git a/docs/zh/operations/system-tables/zookeeper_log.md b/docs/zh/operations/system-tables/zookeeper_log.md
index 59dcdaecdc1..ebc51a2e79d 100644
--- a/docs/zh/operations/system-tables/zookeeper_log.md
+++ b/docs/zh/operations/system-tables/zookeeper_log.md
@@ -15,7 +15,7 @@ slug: /zh/operations/system-tables/zookeeper_log
- `Finalize` — 连接丢失, 未收到响应.
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — 事件发生的日期.
- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — 事件发生的日期和时间.
-- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 用于发出请求的 ZooKeeper 服务器的 IP 地址.
+- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — 用于发出请求的 ZooKeeper 服务器的 IP 地址.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — 用于发出请求的 ZooKeeper 服务器的端口.
- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — ZooKeeper 服务器为每个连接设置的会话 ID.
- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — 会话中请求的 ID. 这通常是一个连续的请求编号. 请求行和配对的 `response`/`finalize` 行相同.
diff --git a/docs/zh/sql-reference/data-types/domains/ipv4.md b/docs/zh/sql-reference/data-types/ipv4.md
similarity index 98%
rename from docs/zh/sql-reference/data-types/domains/ipv4.md
rename to docs/zh/sql-reference/data-types/ipv4.md
index 69e17b2f617..b89af974b87 100644
--- a/docs/zh/sql-reference/data-types/domains/ipv4.md
+++ b/docs/zh/sql-reference/data-types/ipv4.md
@@ -1,5 +1,5 @@
---
-slug: /zh/sql-reference/data-types/domains/ipv4
+slug: /zh/sql-reference/data-types/ipv4
---
## IPv4 {#ipv4}
diff --git a/docs/zh/sql-reference/data-types/domains/ipv6.md b/docs/zh/sql-reference/data-types/ipv6.md
similarity index 98%
rename from docs/zh/sql-reference/data-types/domains/ipv6.md
rename to docs/zh/sql-reference/data-types/ipv6.md
index 9dd88692c37..3896bb873d8 100644
--- a/docs/zh/sql-reference/data-types/domains/ipv6.md
+++ b/docs/zh/sql-reference/data-types/ipv6.md
@@ -1,5 +1,5 @@
---
-slug: /zh/sql-reference/data-types/domains/ipv6
+slug: /zh/sql-reference/data-types/ipv6
---
## IPv6 {#ipv6}
diff --git a/docs/zh/sql-reference/functions/date-time-functions.md b/docs/zh/sql-reference/functions/date-time-functions.md
index 53dadc23c6d..e4b70322477 100644
--- a/docs/zh/sql-reference/functions/date-time-functions.md
+++ b/docs/zh/sql-reference/functions/date-time-functions.md
@@ -643,6 +643,8 @@ date_diff('unit', startdate, enddate, [timezone])
- `unit` — `value`对应的时间单位。类型为[String](../../sql-reference/data-types/string.md)。
可能的值：
+ - `microsecond`
+ - `millisecond`
- `second`
- `minute`
- `hour`
diff --git a/docs/zh/sql-reference/functions/functions-for-nulls.md b/docs/zh/sql-reference/functions/functions-for-nulls.md
index 4dd30970923..b3dca3ac549 100644
--- a/docs/zh/sql-reference/functions/functions-for-nulls.md
+++ b/docs/zh/sql-reference/functions/functions-for-nulls.md
@@ -192,7 +192,7 @@ SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook
**返回值**
- 如果`x`不为`NULL`，返回非`Nullable`类型的原始值。
-- 如果`x`为`NULL`，返回对应非`Nullable`类型的默认值。
+- 如果`x`为`NULL`，则返回任意值。
**示例**
diff --git a/docs/zh/sql-reference/statements/create/view.md b/docs/zh/sql-reference/statements/create/view.md
index 8ce2d20a10c..bce0994ecd2 100644
--- a/docs/zh/sql-reference/statements/create/view.md
+++ b/docs/zh/sql-reference/statements/create/view.md
@@ -72,7 +72,7 @@ ClickHouse 中的物化视图更像是插入触发器。 如果视图查询中
使用[allow_experimental_live_view](../../../operations/settings/settings.md#allow-experimental-live-view)设置启用实时视图和`WATCH`查询的使用。 输入命令`set allow_experimental_live_view = 1`。
```sql
-CREATE LIVE VIEW [IF NOT EXISTS] [db.]table_name [WITH [TIMEOUT [value_in_sec] [AND]] [REFRESH [value_in_sec]]] AS SELECT ...
+CREATE LIVE VIEW [IF NOT EXISTS] [db.]table_name [WITH REFRESH [value_in_sec]] AS SELECT ...
```
实时视图存储相应[SELECT](../../../sql-reference/statements/select/index.md)查询的结果，并在查询结果更改时随时更新。 查询结果以及与新数据结合所需的部分结果存储在内存中，为重复查询提供更高的性能。当使用[WATCH](../../../sql-reference/statements/watch.md)查询更改查询结果时，实时视图可以提供推送通知。
diff --git a/packages/clickhouse-server.service b/packages/clickhouse-server.service
index 7742d8b278a..42dc5bd380d 100644
--- a/packages/clickhouse-server.service
+++ b/packages/clickhouse-server.service
@@ -29,6 +29,7 @@ EnvironmentFile=-/etc/default/clickhouse
LimitCORE=infinity
LimitNOFILE=500000
CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE
+AmbientCapabilities=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE
[Install]
# ClickHouse should not start from the rescue shell (rescue.target).
diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index 54b091700b2..e73f77819ad 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -4,7 +4,9 @@
#include