thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into kill_scalar_github

Browse Source
This commit is contained in:
mergify[bot] 2021-12-15 09:22:58 +00:00 committed by GitHub
commit bff0424139
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
405 changed files with 10194 additions and 2395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitattributes vendored
View File

@ -1,4 +1,2 @@
contrib/* linguist-vendored contrib/* linguist-vendored
*.h linguist-language=C++ *.h linguist-language=C++
# to avoid frequent conflicts
tests/queries/0_stateless/arcadia_skip_list.txt text merge=union

4
.github/workflows/backport_branches.yml vendored
View File

@ -207,8 +207,6 @@ jobs:
- BuilderDebRelease - BuilderDebRelease
- BuilderDebAsan - BuilderDebAsan
- BuilderDebTsan - BuilderDebTsan
- BuilderDebUBsan
- BuilderDebMsan
- BuilderDebDebug - BuilderDebDebug
runs-on: [self-hosted, style-checker] runs-on: [self-hosted, style-checker]
steps: steps:
@ -333,7 +331,7 @@ jobs:
############################# INTEGRATION TESTS ############################################# ############################# INTEGRATION TESTS #############################################
############################################################################################# #############################################################################################
IntegrationTestsRelease: IntegrationTestsRelease:
needs: [BuilderDebRelease, FunctionalStatelessTestRelease] needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester] runs-on: [self-hosted, stress-tester]
steps: steps:
- name: Download json reports - name: Download json reports

661
.github/workflows/main.yml vendored
View File

@ -199,6 +199,44 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
BuilderPerformance:
needs: [DockerHubPush, FastTest]
runs-on: [self-hosted, builder]
steps:
- name: Download changed images
uses: actions/download-artifact@v2
with:
name: changed_images
path: ${{ runner.temp }}/images_path
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'recursive'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
env:
TEMP_PATH: ${{runner.temp}}/build_check
IMAGES_PATH: ${{runner.temp}}/images_path
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NAME: 'performance'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
BuilderBinRelease: BuilderBinRelease:
needs: [DockerHubPush, FastTest] needs: [DockerHubPush, FastTest]
runs-on: [self-hosted, builder] runs-on: [self-hosted, builder]
@ -799,7 +837,7 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
FunctionalStatelessTestReleaseDatabaseReplicated: FunctionalStatelessTestReleaseDatabaseReplicated0:
needs: [BuilderDebRelease] needs: [BuilderDebRelease]
runs-on: [self-hosted, func-tester] runs-on: [self-hosted, func-tester]
steps: steps:
@ -816,6 +854,39 @@ jobs:
CHECK_NAME: 'Stateless tests (release, DatabaseReplicated, actions)' CHECK_NAME: 'Stateless tests (release, DatabaseReplicated, actions)'
REPO_COPY: ${{runner.temp}}/stateless_database_replicated/ClickHouse REPO_COPY: ${{runner.temp}}/stateless_database_replicated/ClickHouse
KILL_TIMEOUT: 10800 KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 2
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestReleaseDatabaseReplicated1:
needs: [BuilderDebRelease]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_database_replicated
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (release, DatabaseReplicated, actions)'
REPO_COPY: ${{runner.temp}}/stateless_database_replicated/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 2
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -857,7 +928,7 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
FunctionalStatelessTestAsan: FunctionalStatelessTestAsan0:
needs: [BuilderDebAsan] needs: [BuilderDebAsan]
runs-on: [self-hosted, func-tester] runs-on: [self-hosted, func-tester]
steps: steps:
@ -874,6 +945,8 @@ jobs:
CHECK_NAME: 'Stateless tests (address, actions)' CHECK_NAME: 'Stateless tests (address, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT: 10800 KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 2
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -886,7 +959,38 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
FunctionalStatelessTestTsan: FunctionalStatelessTestAsan1:
needs: [BuilderDebAsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_debug
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (address, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 2
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestTsan0:
needs: [BuilderDebTsan] needs: [BuilderDebTsan]
runs-on: [self-hosted, func-tester] runs-on: [self-hosted, func-tester]
steps: steps:
@ -903,6 +1007,70 @@ jobs:
CHECK_NAME: 'Stateless tests (thread, actions)' CHECK_NAME: 'Stateless tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY: ${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT: 10800 KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestTsan1:
needs: [BuilderDebTsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_tsan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestTsan2:
needs: [BuilderDebTsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_tsan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 3
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -944,7 +1112,7 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
FunctionalStatelessTestMsan: FunctionalStatelessTestMsan0:
needs: [BuilderDebMsan] needs: [BuilderDebMsan]
runs-on: [self-hosted, func-tester] runs-on: [self-hosted, func-tester]
steps: steps:
@ -961,6 +1129,8 @@ jobs:
CHECK_NAME: 'Stateless tests (memory, actions)' CHECK_NAME: 'Stateless tests (memory, actions)'
REPO_COPY: ${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY: ${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT: 10800 KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 3
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -973,7 +1143,69 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
FunctionalStatelessTestDebug: FunctionalStatelessTestMsan1:
needs: [BuilderDebMsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_memory
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (memory, actions)'
REPO_COPY: ${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestMsan2:
needs: [BuilderDebMsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_memory
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (memory, actions)'
REPO_COPY: ${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestDebug0:
needs: [BuilderDebDebug] needs: [BuilderDebDebug]
runs-on: [self-hosted, func-tester] runs-on: [self-hosted, func-tester]
steps: steps:
@ -990,6 +1222,70 @@ jobs:
CHECK_NAME: 'Stateless tests (debug, actions)' CHECK_NAME: 'Stateless tests (debug, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT: 10800 KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestDebug1:
needs: [BuilderDebDebug]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_debug
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (debug, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestDebug2:
needs: [BuilderDebDebug]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_debug
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (debug, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 3
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -1497,8 +1793,8 @@ jobs:
############################################################################################# #############################################################################################
############################# INTEGRATION TESTS ############################################# ############################# INTEGRATION TESTS #############################################
############################################################################################# #############################################################################################
IntegrationTestsAsan: IntegrationTestsAsan0:
needs: [BuilderDebAsan, FunctionalStatelessTestAsan] needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester] runs-on: [self-hosted, stress-tester]
steps: steps:
- name: Download json reports - name: Download json reports
@ -1513,6 +1809,8 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (asan, actions)' CHECK_NAME: 'Integration tests (asan, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 3
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -1525,8 +1823,68 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
IntegrationTestsTsan: IntegrationTestsAsan1:
needs: [BuilderDebTsan, FunctionalStatelessTestTsan] needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_asan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (asan, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsAsan2:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_asan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (asan, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsTsan0:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester] runs-on: [self-hosted, stress-tester]
steps: steps:
- name: Download json reports - name: Download json reports
@ -1541,6 +1899,8 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (thread, actions)' CHECK_NAME: 'Integration tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY: ${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 4
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -1553,8 +1913,98 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
IntegrationTestsRelease: IntegrationTestsTsan1:
needs: [BuilderDebRelease, FunctionalStatelessTestRelease] needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_tsan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsTsan2:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_tsan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsTsan3:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_tsan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM: 3
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsRelease0:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester] runs-on: [self-hosted, stress-tester]
steps: steps:
- name: Download json reports - name: Download json reports
@ -1569,6 +2019,38 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (release, actions)' CHECK_NAME: 'Integration tests (release, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_release/ClickHouse REPO_COPY: ${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 2
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsRelease1:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_release
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (release, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 2
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -1752,6 +2234,129 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
#############################################################################################
#################################### PERFORMANCE TESTS ######################################
#############################################################################################
PerformanceComparison0:
needs: [BuilderPerformance]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
env:
TEMP_PATH: ${{runner.temp}}/performance_comparison
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Performance Comparison (actions)'
REPO_COPY: ${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
PerformanceComparison1:
needs: [BuilderPerformance]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
env:
TEMP_PATH: ${{runner.temp}}/performance_comparison
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Performance Comparison (actions)'
REPO_COPY: ${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
PerformanceComparison2:
needs: [BuilderPerformance]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
env:
TEMP_PATH: ${{runner.temp}}/performance_comparison
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Performance Comparison (actions)'
REPO_COPY: ${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
PerformanceComparison3:
needs: [BuilderPerformance]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
env:
TEMP_PATH: ${{runner.temp}}/performance_comparison
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Performance Comparison (actions)'
REPO_COPY: ${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM: 3
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FinishCheck: FinishCheck:
needs: needs:
- StyleCheck - StyleCheck
@ -1759,13 +2364,21 @@ jobs:
- CheckLabels - CheckLabels
- BuilderReport - BuilderReport
- FastTest - FastTest
- FunctionalStatelessTestDebug - FunctionalStatelessTestDebug0
- FunctionalStatelessTestDebug1
- FunctionalStatelessTestDebug2
- FunctionalStatelessTestRelease - FunctionalStatelessTestRelease
- FunctionalStatelessTestReleaseDatabaseReplicated - FunctionalStatelessTestReleaseDatabaseReplicated0
- FunctionalStatelessTestReleaseDatabaseReplicated1
- FunctionalStatelessTestReleaseWideParts - FunctionalStatelessTestReleaseWideParts
- FunctionalStatelessTestAsan - FunctionalStatelessTestAsan0
- FunctionalStatelessTestTsan - FunctionalStatelessTestAsan1
- FunctionalStatelessTestMsan - FunctionalStatelessTestTsan0
- FunctionalStatelessTestTsan1
- FunctionalStatelessTestTsan2
- FunctionalStatelessTestMsan0
- FunctionalStatelessTestMsan1
- FunctionalStatelessTestMsan2
- FunctionalStatelessTestUBsan - FunctionalStatelessTestUBsan
- FunctionalStatefulTestDebug - FunctionalStatefulTestDebug
- FunctionalStatefulTestRelease - FunctionalStatefulTestRelease
@ -1783,9 +2396,19 @@ jobs:
- ASTFuzzerTestTsan - ASTFuzzerTestTsan
- ASTFuzzerTestMSan - ASTFuzzerTestMSan
- ASTFuzzerTestUBSan - ASTFuzzerTestUBSan
- IntegrationTestsAsan - IntegrationTestsAsan0
- IntegrationTestsRelease - IntegrationTestsAsan1
- IntegrationTestsTsan - IntegrationTestsAsan2
- IntegrationTestsRelease0
- IntegrationTestsRelease1
- IntegrationTestsTsan0
- IntegrationTestsTsan1
- IntegrationTestsTsan2
- IntegrationTestsTsan3
- PerformanceComparison0
- PerformanceComparison1
- PerformanceComparison2
- PerformanceComparison3
- PVSCheck - PVSCheck
- UnitTestsAsan - UnitTestsAsan
- UnitTestsTsan - UnitTestsTsan

625
.github/workflows/master.yml vendored
View File

@ -135,6 +135,44 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
BuilderPerformance:
needs: DockerHubPush
runs-on: [self-hosted, builder]
steps:
- name: Download changed images
uses: actions/download-artifact@v2
with:
name: changed_images
path: ${{ runner.temp }}/images_path
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'recursive'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
env:
TEMP_PATH: ${{runner.temp}}/build_check
IMAGES_PATH: ${{runner.temp}}/images_path
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NAME: 'performance'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
BuilderBinRelease: BuilderBinRelease:
needs: [DockerHubPush] needs: [DockerHubPush]
if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }} if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
@ -770,7 +808,7 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
FunctionalStatelessTestAsan: FunctionalStatelessTestAsan0:
needs: [BuilderDebAsan] needs: [BuilderDebAsan]
runs-on: [self-hosted, func-tester] runs-on: [self-hosted, func-tester]
steps: steps:
@ -787,6 +825,8 @@ jobs:
CHECK_NAME: 'Stateless tests (address, actions)' CHECK_NAME: 'Stateless tests (address, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT: 10800 KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 2
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -799,7 +839,38 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
FunctionalStatelessTestTsan: FunctionalStatelessTestAsan1:
needs: [BuilderDebAsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_debug
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (address, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 2
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestTsan0:
needs: [BuilderDebTsan] needs: [BuilderDebTsan]
runs-on: [self-hosted, func-tester] runs-on: [self-hosted, func-tester]
steps: steps:
@ -816,6 +887,70 @@ jobs:
CHECK_NAME: 'Stateless tests (thread, actions)' CHECK_NAME: 'Stateless tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY: ${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT: 10800 KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestTsan1:
needs: [BuilderDebTsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_tsan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestTsan2:
needs: [BuilderDebTsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_tsan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 3
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -857,7 +992,7 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
FunctionalStatelessTestMsan: FunctionalStatelessTestMsan0:
needs: [BuilderDebMsan] needs: [BuilderDebMsan]
runs-on: [self-hosted, func-tester] runs-on: [self-hosted, func-tester]
steps: steps:
@ -874,6 +1009,8 @@ jobs:
CHECK_NAME: 'Stateless tests (memory, actions)' CHECK_NAME: 'Stateless tests (memory, actions)'
REPO_COPY: ${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY: ${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT: 10800 KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 3
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -886,7 +1023,69 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
FunctionalStatelessTestDebug: FunctionalStatelessTestMsan1:
needs: [BuilderDebMsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_memory
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (memory, actions)'
REPO_COPY: ${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestMsan2:
needs: [BuilderDebMsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_memory
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (memory, actions)'
REPO_COPY: ${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestDebug0:
needs: [BuilderDebDebug] needs: [BuilderDebDebug]
runs-on: [self-hosted, func-tester] runs-on: [self-hosted, func-tester]
steps: steps:
@ -903,6 +1102,70 @@ jobs:
CHECK_NAME: 'Stateless tests (debug, actions)' CHECK_NAME: 'Stateless tests (debug, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT: 10800 KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestDebug1:
needs: [BuilderDebDebug]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_debug
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (debug, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestDebug2:
needs: [BuilderDebDebug]
runs-on: [self-hosted, func-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Functional test
env:
TEMP_PATH: ${{runner.temp}}/stateless_debug
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (debug, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT: 10800
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 3
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -1267,8 +1530,8 @@ jobs:
############################################################################################# #############################################################################################
############################# INTEGRATION TESTS ############################################# ############################# INTEGRATION TESTS #############################################
############################################################################################# #############################################################################################
IntegrationTestsAsan: IntegrationTestsAsan0:
needs: [BuilderDebAsan, FunctionalStatelessTestAsan] needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester] runs-on: [self-hosted, stress-tester]
steps: steps:
- name: Download json reports - name: Download json reports
@ -1283,6 +1546,8 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (asan, actions)' CHECK_NAME: 'Integration tests (asan, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 3
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -1295,8 +1560,68 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
IntegrationTestsTsan: IntegrationTestsAsan1:
needs: [BuilderDebTsan, FunctionalStatelessTestTsan] needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_asan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (asan, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsAsan2:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_asan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (asan, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsTsan0:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester] runs-on: [self-hosted, stress-tester]
steps: steps:
- name: Download json reports - name: Download json reports
@ -1311,6 +1636,8 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (thread, actions)' CHECK_NAME: 'Integration tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY: ${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 4
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -1323,8 +1650,98 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
IntegrationTestsRelease: IntegrationTestsTsan1:
needs: [BuilderDebRelease, FunctionalStatelessTestRelease] needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_tsan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsTsan2:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_tsan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsTsan3:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_tsan
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (thread, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM: 3
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsRelease0:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester] runs-on: [self-hosted, stress-tester]
steps: steps:
- name: Download json reports - name: Download json reports
@ -1339,6 +1756,38 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (release, actions)' CHECK_NAME: 'Integration tests (release, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_release/ClickHouse REPO_COPY: ${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 2
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsRelease1:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_release
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests (release, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 2
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH mkdir -p $TEMP_PATH
@ -1623,7 +2072,7 @@ jobs:
env: env:
TEMP_PATH: ${{runner.temp}}/unit_tests_ubsan TEMP_PATH: ${{runner.temp}}/unit_tests_ubsan
REPORTS_PATH: ${{runner.temp}}/reports_dir REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Unit tests (msan, actions)' CHECK_NAME: 'Unit tests (ubsan, actions)'
REPO_COPY: ${{runner.temp}}/unit_tests_ubsan/ClickHouse REPO_COPY: ${{runner.temp}}/unit_tests_ubsan/ClickHouse
run: | run: |
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
@ -1637,16 +2086,146 @@ jobs:
docker kill $(docker ps -q) ||: docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||: docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH sudo rm -fr $TEMP_PATH
#############################################################################################
#################################### PERFORMANCE TESTS ######################################
#############################################################################################
PerformanceComparison0:
needs: [BuilderPerformance]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
env:
TEMP_PATH: ${{runner.temp}}/performance_comparison
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Performance Comparison (actions)'
REPO_COPY: ${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM: 0
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
PerformanceComparison1:
needs: [BuilderPerformance]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
env:
TEMP_PATH: ${{runner.temp}}/performance_comparison
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Performance Comparison (actions)'
REPO_COPY: ${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM: 1
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
PerformanceComparison2:
needs: [BuilderPerformance]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
env:
TEMP_PATH: ${{runner.temp}}/performance_comparison
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Performance Comparison (actions)'
REPO_COPY: ${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM: 2
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
PerformanceComparison3:
needs: [BuilderPerformance]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
env:
TEMP_PATH: ${{runner.temp}}/performance_comparison
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Performance Comparison (actions)'
REPO_COPY: ${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM: 3
RUN_BY_HASH_TOTAL: 4
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
FinishCheck: FinishCheck:
needs: needs:
- DockerHubPush - DockerHubPush
- BuilderReport - BuilderReport
- FunctionalStatelessTestDebug - FunctionalStatelessTestDebug0
- FunctionalStatelessTestDebug1
- FunctionalStatelessTestDebug2
- FunctionalStatelessTestRelease - FunctionalStatelessTestRelease
- FunctionalStatelessTestReleaseDatabaseOrdinary - FunctionalStatelessTestReleaseDatabaseOrdinary
- FunctionalStatelessTestAsan - FunctionalStatelessTestAsan0
- FunctionalStatelessTestTsan - FunctionalStatelessTestAsan1
- FunctionalStatelessTestMsan - FunctionalStatelessTestTsan0
- FunctionalStatelessTestTsan1
- FunctionalStatelessTestTsan2
- FunctionalStatelessTestMsan0
- FunctionalStatelessTestMsan1
- FunctionalStatelessTestMsan2
- FunctionalStatelessTestUBsan - FunctionalStatelessTestUBsan
- FunctionalStatefulTestDebug - FunctionalStatefulTestDebug
- FunctionalStatefulTestRelease - FunctionalStatefulTestRelease
@ -1660,9 +2239,19 @@ jobs:
- StressTestTsan - StressTestTsan
- StressTestMsan - StressTestMsan
- StressTestUBsan - StressTestUBsan
- IntegrationTestsAsan - IntegrationTestsAsan0
- IntegrationTestsRelease - IntegrationTestsAsan1
- IntegrationTestsTsan - IntegrationTestsAsan2
- IntegrationTestsRelease0
- IntegrationTestsRelease1
- IntegrationTestsTsan0
- IntegrationTestsTsan1
- IntegrationTestsTsan2
- IntegrationTestsTsan3
- PerformanceComparison0
- PerformanceComparison1
- PerformanceComparison2
- PerformanceComparison3
- CompatibilityCheck - CompatibilityCheck
- ASTFuzzerTestDebug - ASTFuzzerTestDebug
- ASTFuzzerTestAsan - ASTFuzzerTestAsan

3
.gitmodules vendored
View File

@ -247,3 +247,6 @@
[submodule "contrib/sysroot"] [submodule "contrib/sysroot"]
path = contrib/sysroot path = contrib/sysroot
url = https://github.com/ClickHouse-Extras/sysroot.git url = https://github.com/ClickHouse-Extras/sysroot.git
[submodule "contrib/azure"]
path = contrib/azure
url = https://github.com/ClickHouse-Extras/azure-sdk-for-cpp.git

178
CHANGELOG.md
View File

@ -1,3 +1,181 @@
### ClickHouse release v21.12, 2021-12-15
#### Backward Incompatible Change
* *A fix for a feature that previously had unwanted behaviour.* Do not allow direct select for Kafka/RabbitMQ/FileLog. Can be enabled by setting `stream_like_engine_allow_direct_select`. Direct select will be not allowed even if enabled by setting, in case there is an attached materialized view. For Kafka and RabbitMQ direct selectm if allowed, will not commit massages by default. To enable commits with direct select, user must use storage level setting `kafka{rabbitmq}_commit_on_select=1` (default `0`). [#31053](https://github.com/ClickHouse/ClickHouse/pull/31053) ([Kseniia Sumarokova](https://github.com/kssenii)).
* *A slight change in behaviour of a new function.* Return unquoted string in JSON_VALUE. Closes [#27965](https://github.com/ClickHouse/ClickHouse/issues/27965). [#31008](https://github.com/ClickHouse/ClickHouse/pull/31008) ([Kseniia Sumarokova](https://github.com/kssenii)).
* *Setting rename.* Add custom null representation support for TSV/CSV input formats. Fix deserialing Nullable(String) in TSV/CSV/JSONCompactStringsEachRow/JSONStringsEachRow input formats. Rename `output_format_csv_null_representation` and `output_format_tsv_null_representation` to `format_csv_null_representation` and `format_tsv_null_representation` accordingly. [#30497](https://github.com/ClickHouse/ClickHouse/pull/30497) ([Kruglov Pavel](https://github.com/Avogar)).
* *Further deprecation of already unused code.* This is relevant only for users of ClickHouse versions older than 20.6. A "leader election" mechanism is removed from `ReplicatedMergeTree`, because multiple leaders are supported since 20.6. If you are upgrading from an older version and some replica with an old version is a leader, then server will fail to start after upgrade. Stop replicas with old version to make new version start. After that it will not be possible to downgrade to version older than 20.6. [#32140](https://github.com/ClickHouse/ClickHouse/pull/32140) ([tavplubix](https://github.com/tavplubix)).
#### New Feature
* Implemented more of the ZooKeeper Four Letter Words commands in clickhouse-keeper: https://zookeeper.apache.org/doc/r3.4.8/zookeeperAdmin.html#sc_zkCommands. [#28981](https://github.com/ClickHouse/ClickHouse/pull/28981) ([JackyWoo](https://github.com/JackyWoo)). Now `clickhouse-keeper` is feature complete.
* Support for `Bool` data type. [#31072](https://github.com/ClickHouse/ClickHouse/pull/31072) ([kevin wan](https://github.com/MaxWk)).
* Support for `PARTITION BY` in File, URL, HDFS storages and with `INSERT INTO` table function. Closes [#30273](https://github.com/ClickHouse/ClickHouse/issues/30273). [#30690](https://github.com/ClickHouse/ClickHouse/pull/30690) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Added `CONSTRAINT ... ASSUME ...` (without checking during `INSERT`). Added query transformation to CNF (https://github.com/ClickHouse/ClickHouse/issues/11749) for more convenient optimization. Added simple query rewriting using constraints (only simple matching now, will be improved to support <,=,>... later). Added ability to replace heavy columns with light columns if it's possible. [#18787](https://github.com/ClickHouse/ClickHouse/pull/18787) ([Nikita Vasilev](https://github.com/nikvas0)).
* Basic access authentication for http/url functions. [#31648](https://github.com/ClickHouse/ClickHouse/pull/31648) ([michael1589](https://github.com/michael1589)).
* Support `INTERVAL` type in `STEP` clause for `WITH FILL` modifier. [#30927](https://github.com/ClickHouse/ClickHouse/pull/30927) ([Anton Popov](https://github.com/CurtizJ)).
* Add support for parallel reading from multiple files and support globs in `FROM INFILE` clause. [#30135](https://github.com/ClickHouse/ClickHouse/pull/30135) ([Filatenkov Artur](https://github.com/FArthur-cmd)).
* Add support for `Identifier` table and database query parameters. Closes [#27226](https://github.com/ClickHouse/ClickHouse/issues/27226). [#28668](https://github.com/ClickHouse/ClickHouse/pull/28668) ([Nikolay Degterinsky](https://github.com/evillique)).
* *TLDR: Major improvements of completeness and consistency of text formats.* Refactor formats `TSV`, `TSVRaw`, `CSV` and `JSONCompactEachRow`, `JSONCompactStringsEachRow`, remove code duplication, add base interface for formats with `-WithNames` and `-WithNamesAndTypes` suffixes. Add formats `CSVWithNamesAndTypes`, `TSVRawWithNames`, `TSVRawWithNamesAndTypes`, `JSONCompactEachRowWIthNames`, `JSONCompactStringsEachRowWIthNames`, `RowBinaryWithNames`. Support parallel parsing for formats `TSVWithNamesAndTypes`, `TSVRaw(WithNames/WIthNamesAndTypes)`, `CSVWithNamesAndTypes`, `JSONCompactEachRow(WithNames/WIthNamesAndTypes)`, `JSONCompactStringsEachRow(WithNames/WIthNamesAndTypes)`. Support columns mapping and types checking for `RowBinaryWithNamesAndTypes` format. Add setting `input_format_with_types_use_header` which specify if we should check that types written in <format_name>`WIthNamesAndTypes` format matches with table structure. Add setting `input_format_csv_empty_as_default` and use it in CSV format instead of `input_format_defaults_for_omitted_fields` (because this setting should not control `csv_empty_as_default`). Fix usage of setting `input_format_defaults_for_omitted_fields` (it was used only as `csv_empty_as_default`, but it should control calculation of default expressions for omitted fields). Fix Nullable input/output in `TSVRaw` format, make this format fully compatible with inserting into TSV. Fix inserting NULLs in `LowCardinality(Nullable)` when `input_format_null_as_default` is enabled (previously default values was inserted instead of actual NULLs). Fix strings deserialization in `JSONStringsEachRow`/`JSONCompactStringsEachRow` formats (strings were parsed just until first '\n' or '\t'). Add ability to use `Raw` escaping rule in Template input format. Add diagnostic info for JSONCompactEachRow(WithNames/WIthNamesAndTypes) input format. Fix bug with parallel parsing of `-WithNames` formats in case when setting `min_chunk_bytes_for_parallel_parsing` is less than bytes in a single row. [#30178](https://github.com/ClickHouse/ClickHouse/pull/30178) ([Kruglov Pavel](https://github.com/Avogar)). Allow to print/parse names and types of colums in `CustomSeparated` input/output format. Add formats `CustomSeparatedWithNames/WithNamesAndTypes` similar to `TSVWithNames/WithNamesAndTypes`. [#31434](https://github.com/ClickHouse/ClickHouse/pull/31434) ([Kruglov Pavel](https://github.com/Avogar)).
* Aliyun OSS Storage support. [#31286](https://github.com/ClickHouse/ClickHouse/pull/31286) ([cfcz48](https://github.com/cfcz48)).
* Exposes all settings of the global thread pool in the configuration file. [#31285](https://github.com/ClickHouse/ClickHouse/pull/31285) ([Tomáš Hromada](https://github.com/gyfis)).
* Introduced window functions `exponentialTimeDecayedSum`, `exponentialTimeDecayedMax`, `exponentialTimeDecayedCount` and `exponentialTimeDecayedAvg` which are more effective than `exponentialMovingAverage` for bigger windows. Also more use-cases were covered. [#29799](https://github.com/ClickHouse/ClickHouse/pull/29799) ([Vladimir Chebotarev](https://github.com/excitoon)).
* Add option to compress logs before writing them to a file using LZ4. Closes [#23860](https://github.com/ClickHouse/ClickHouse/issues/23860). [#29219](https://github.com/ClickHouse/ClickHouse/pull/29219) ([Nikolay Degterinsky](https://github.com/evillique)).
* Support `JOIN ON 1 = 1` that have CROSS JOIN semantic. This closes [#25578](https://github.com/ClickHouse/ClickHouse/issues/25578). [#25894](https://github.com/ClickHouse/ClickHouse/pull/25894) ([Vladimir C](https://github.com/vdimir)).
* Add Map combinator for `Map` type. - Rename old `sum-, min-, max- Map` for mapped arrays to `sum-, min-, max- MappedArrays`. [#24539](https://github.com/ClickHouse/ClickHouse/pull/24539) ([Ildus Kurbangaliev](https://github.com/ildus)).
* Make reading from HTTP retriable. Closes [#29696](https://github.com/ClickHouse/ClickHouse/issues/29696). [#29894](https://github.com/ClickHouse/ClickHouse/pull/29894) ([Kseniia Sumarokova](https://github.com/kssenii)).
#### Experimental Feature
* `WINDOW VIEW` to enable stream processing in ClickHouse. [#8331](https://github.com/ClickHouse/ClickHouse/pull/8331) ([vxider](https://github.com/Vxider)).
* Drop support for using Ordinary databases with `MaterializedMySQL`. [#31292](https://github.com/ClickHouse/ClickHouse/pull/31292) ([Stig Bakken](https://github.com/stigsb)).
* Implement the commands BACKUP and RESTORE for the Log family. This feature is under development. [#30688](https://github.com/ClickHouse/ClickHouse/pull/30688) ([Vitaly Baranov](https://github.com/vitlibar)).
#### Performance Improvement
* Reduce memory usage when reading with `s3` / `url` / `hdfs` formats `Parquet`, `ORC`, `Arrow` (controlled by setting `input_format_allow_seeks`, enabled by default). Also add setting `remote_read_min_bytes_for_seek` to control seeks. Closes [#10461](https://github.com/ClickHouse/ClickHouse/issues/10461). Closes [#16857](https://github.com/ClickHouse/ClickHouse/issues/16857). [#30936](https://github.com/ClickHouse/ClickHouse/pull/30936) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Add optimizations for constant conditions in JOIN ON, ref [#26928](https://github.com/ClickHouse/ClickHouse/issues/26928). [#27021](https://github.com/ClickHouse/ClickHouse/pull/27021) ([Vladimir C](https://github.com/vdimir)).
* Support parallel formatting for all text formats, except `JSONEachRowWithProgress` and `PrettyCompactMonoBlock`. [#31489](https://github.com/ClickHouse/ClickHouse/pull/31489) ([Kruglov Pavel](https://github.com/Avogar)).
* Speed up count over nullable columns. [#31806](https://github.com/ClickHouse/ClickHouse/pull/31806) ([Raúl Marín](https://github.com/Algunenano)).
* Speed up `avg` and `sumCount` aggregate functions. [#31694](https://github.com/ClickHouse/ClickHouse/pull/31694) ([Raúl Marín](https://github.com/Algunenano)).
* Improve performance of JSON and XML output formats. [#31673](https://github.com/ClickHouse/ClickHouse/pull/31673) ([alexey-milovidov](https://github.com/alexey-milovidov)).
* Improve performance of syncing data to block device. This closes [#31181](https://github.com/ClickHouse/ClickHouse/issues/31181). [#31229](https://github.com/ClickHouse/ClickHouse/pull/31229) ([zhanglistar](https://github.com/zhanglistar)).
* Fixing query performance issue in `LiveView` tables. Fixes [#30831](https://github.com/ClickHouse/ClickHouse/issues/30831). [#31006](https://github.com/ClickHouse/ClickHouse/pull/31006) ([vzakaznikov](https://github.com/vzakaznikov)).
* Speed up query parsing. [#31949](https://github.com/ClickHouse/ClickHouse/pull/31949) ([Raúl Marín](https://github.com/Algunenano)).
* Allow to split `GraphiteMergeTree` rollup rules for plain/tagged metrics (optional `rule_type` field). [#25122](https://github.com/ClickHouse/ClickHouse/pull/25122) ([Michail Safronov](https://github.com/msaf1980)).
* Remove excessive `DESC TABLE` requests for `remote()` (in case of `remote('127.1', system.one)` (i.e. identifier as the db.table instead of string) there was excessive `DESC TABLE` request). [#32019](https://github.com/ClickHouse/ClickHouse/pull/32019) ([Azat Khuzhin](https://github.com/azat)).
* Optimize function `tupleElement` to reading of subcolumn with enabled setting `optimize_functions_to_subcolumns`. [#31261](https://github.com/ClickHouse/ClickHouse/pull/31261) ([Anton Popov](https://github.com/CurtizJ)).
* Optimize function `mapContains` to reading of subcolumn `key` with enabled settings `optimize_functions_to_subcolumns`. [#31218](https://github.com/ClickHouse/ClickHouse/pull/31218) ([Anton Popov](https://github.com/CurtizJ)).
* Add settings `merge_tree_min_rows_for_concurrent_read_for_remote_filesystem` and `merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem`. [#30970](https://github.com/ClickHouse/ClickHouse/pull/30970) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Skipping mutations of different partitions in `StorageMergeTree`. [#21326](https://github.com/ClickHouse/ClickHouse/pull/21326) ([Vladimir Chebotarev](https://github.com/excitoon)).
#### Improvement
* Do not allow to drop a table or dictionary if some tables or dictionaries depend on it. [#30977](https://github.com/ClickHouse/ClickHouse/pull/30977) ([tavplubix](https://github.com/tavplubix)).
* Allow versioning of aggregate function states. Now we can introduce backward compatible changes in serialization format of aggregate function states. Closes [#12552](https://github.com/ClickHouse/ClickHouse/issues/12552). [#24820](https://github.com/ClickHouse/ClickHouse/pull/24820) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Support PostgreSQL style `ALTER MODIFY COLUMN` syntax. [#32003](https://github.com/ClickHouse/ClickHouse/pull/32003) ([SuperDJY](https://github.com/cmsxbc)).
* Added `update_field` support for `RangeHashedDictionary`, `ComplexKeyRangeHashedDictionary`. [#32185](https://github.com/ClickHouse/ClickHouse/pull/32185) ([Maksim Kita](https://github.com/kitaisreal)).
* The `murmurHash3_128` and `sipHash128` functions now accept an arbitrary number of arguments. This closes [#28774](https://github.com/ClickHouse/ClickHouse/issues/28774). [#28965](https://github.com/ClickHouse/ClickHouse/pull/28965) ([小路](https://github.com/nicelulu)).
* Support default expression for `HDFS` storage and optimize fetching when source is column oriented. [#32256](https://github.com/ClickHouse/ClickHouse/pull/32256) ([李扬](https://github.com/taiyang-li)).
* Improve the operation name of an opentelemetry span. [#32234](https://github.com/ClickHouse/ClickHouse/pull/32234) ([Frank Chen](https://github.com/FrankChen021)).
* Use `Content-Type: application/x-ndjson` (http://ndjson.org/) for output format `JSONEachRow`. [#32223](https://github.com/ClickHouse/ClickHouse/pull/32223) ([Dmitriy Dorofeev](https://github.com/deem0n)).
* Improve skipping unknown fields with quoted escaping rule in Template/CustomSeparated formats. Previously you could skip only quoted strings, now you can skip values with any type. [#32204](https://github.com/ClickHouse/ClickHouse/pull/32204) ([Kruglov Pavel](https://github.com/Avogar)).
* Now `clickhouse-keeper` refuses to start or apply configuration changes when they contain duplicated IDs or endpoints. Fixes [#31339](https://github.com/ClickHouse/ClickHouse/issues/31339). [#32121](https://github.com/ClickHouse/ClickHouse/pull/32121) ([alesapin](https://github.com/alesapin)).
* Set Content-Type in HTTP packets issued from URL engine. [#32113](https://github.com/ClickHouse/ClickHouse/pull/32113) ([Frank Chen](https://github.com/FrankChen021)).
* Return Content-Type as 'application/json' for `JSONEachRow` format if `output_format_json_array_of_rows` is enabled. [#32112](https://github.com/ClickHouse/ClickHouse/pull/32112) ([Frank Chen](https://github.com/FrankChen021)).
* Allow to parse `+` before `Float32`/`Float64` values. [#32079](https://github.com/ClickHouse/ClickHouse/pull/32079) ([Kruglov Pavel](https://github.com/Avogar)).
* Allow a user configured `hdfs_replication` parameter for `DiskHDFS` and `StorageHDFS`. Closes [#32039](https://github.com/ClickHouse/ClickHouse/issues/32039). [#32049](https://github.com/ClickHouse/ClickHouse/pull/32049) ([leosunli](https://github.com/leosunli)).
* Added ClickHouse `exception` and `exception_code` fields to opentelemetry span log. [#32040](https://github.com/ClickHouse/ClickHouse/pull/32040) ([Frank Chen](https://github.com/FrankChen021)).
* Improve opentelemetry span log duration - it was is zero at the query level if there is a query exception. [#32038](https://github.com/ClickHouse/ClickHouse/pull/32038) ([Frank Chen](https://github.com/FrankChen021)).
* Fix the issue that `LowCardinality` of `Int256` cannot be created. [#31832](https://github.com/ClickHouse/ClickHouse/pull/31832) ([alexey-milovidov](https://github.com/alexey-milovidov)).
* Recreate `system.*_log` tables in case of different engine/partition_by. [#31824](https://github.com/ClickHouse/ClickHouse/pull/31824) ([Azat Khuzhin](https://github.com/azat)).
* `MaterializedMySQL`: Fix issue with table named 'table'. [#31781](https://github.com/ClickHouse/ClickHouse/pull/31781) ([Håvard Kvålen](https://github.com/havardk)).
* ClickHouse dictionary source: support named collections. Closes [#31705](https://github.com/ClickHouse/ClickHouse/issues/31705). [#31749](https://github.com/ClickHouse/ClickHouse/pull/31749) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Allow to use named collections configuration for Kafka and RabbitMQ engines (the same way as for other integration table engines). [#31691](https://github.com/ClickHouse/ClickHouse/pull/31691) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Always re-render prompt while navigating history in clickhouse-client. This will improve usability of manipulating very long queries that don't fit on screen. [#31675](https://github.com/ClickHouse/ClickHouse/pull/31675) ([alexey-milovidov](https://github.com/alexey-milovidov)) (author: Amos Bird).
* Add key bindings for navigating through history (instead of lines/history). [#31641](https://github.com/ClickHouse/ClickHouse/pull/31641) ([Azat Khuzhin](https://github.com/azat)).
* Improve the `max_execution_time` checks. Fixed some cases when timeout checks do not happen and query could run too long. [#31636](https://github.com/ClickHouse/ClickHouse/pull/31636) ([Raúl Marín](https://github.com/Algunenano)).
* Better exception message when `users.xml` cannot be loaded due to bad password hash. This closes [#24126](https://github.com/ClickHouse/ClickHouse/issues/24126). [#31557](https://github.com/ClickHouse/ClickHouse/pull/31557) ([Vitaly Baranov](https://github.com/vitlibar)).
* Use shard and replica name from `Replicated` database arguments when expanding macros in `ReplicatedMergeTree` arguments if these macros are not defined in config. Closes [#31471](https://github.com/ClickHouse/ClickHouse/issues/31471). [#31488](https://github.com/ClickHouse/ClickHouse/pull/31488) ([tavplubix](https://github.com/tavplubix)).
* Better analysis for `min/max/count` projection. Now, with enabled `allow_experimental_projection_optimization`, virtual `min/max/count` projection can be used together with columns from partition key. [#31474](https://github.com/ClickHouse/ClickHouse/pull/31474) ([Amos Bird](https://github.com/amosbird)).
* Add `--pager` support for `clickhouse-local`. [#31457](https://github.com/ClickHouse/ClickHouse/pull/31457) ([Azat Khuzhin](https://github.com/azat)).
* Fix waiting of the editor during interactive query edition (`waitpid()` returns -1 on `SIGWINCH` and `EDITOR` and `clickhouse-local`/`clickhouse-client` works concurrently). [#31456](https://github.com/ClickHouse/ClickHouse/pull/31456) ([Azat Khuzhin](https://github.com/azat)).
* Throw an exception if there is some garbage after field in `JSONCompactStrings(EachRow)` format. [#31455](https://github.com/ClickHouse/ClickHouse/pull/31455) ([Kruglov Pavel](https://github.com/Avogar)).
* Default value of `http_send_timeout` and `http_receive_timeout` settings changed from 1800 (30 minutes) to 180 (3 minutes). [#31450](https://github.com/ClickHouse/ClickHouse/pull/31450) ([tavplubix](https://github.com/tavplubix)).
* `MaterializedMySQL` now handles `CREATE TABLE ... LIKE ...` DDL queries. [#31410](https://github.com/ClickHouse/ClickHouse/pull/31410) ([Stig Bakken](https://github.com/stigsb)).
* Return artificial create query when executing `show create table` on system's tables. [#31391](https://github.com/ClickHouse/ClickHouse/pull/31391) ([SuperDJY](https://github.com/cmsxbc)).
* Previously progress was shown only for `numbers` table function. Now for `numbers_mt` it is also shown. [#31318](https://github.com/ClickHouse/ClickHouse/pull/31318) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Initial user's roles are used now to find row policies, see [#31080](https://github.com/ClickHouse/ClickHouse/issues/31080). [#31262](https://github.com/ClickHouse/ClickHouse/pull/31262) ([Vitaly Baranov](https://github.com/vitlibar)).
* If some obsolete setting is changed - show warning in `system.warnings`. [#31252](https://github.com/ClickHouse/ClickHouse/pull/31252) ([tavplubix](https://github.com/tavplubix)).
* Improved backoff for background cleanup tasks in `MergeTree`. Settings `merge_tree_clear_old_temporary_directories_interval_seconds` and `merge_tree_clear_old_parts_interval_seconds` moved from users settings to merge tree settings. [#31180](https://github.com/ClickHouse/ClickHouse/pull/31180) ([tavplubix](https://github.com/tavplubix)).
* Now every replica will send to client only incremental information about profile events counters. [#31155](https://github.com/ClickHouse/ClickHouse/pull/31155) ([Dmitry Novik](https://github.com/novikd)). This makes `--hardware_utilization` option in `clickhouse-client` usable.
* Enable multiline editing in clickhouse-client by default. This addresses [#31121](https://github.com/ClickHouse/ClickHouse/issues/31121) . [#31123](https://github.com/ClickHouse/ClickHouse/pull/31123) ([Amos Bird](https://github.com/amosbird)).
* Function name normalization for `ALTER` queries. This helps avoid metadata mismatch between creating table with indices/projections and adding indices/projections via alter commands. This is a follow-up PR of https://github.com/ClickHouse/ClickHouse/pull/20174. Mark as improvements as there are no bug reports and the senario is somehow rare. [#31095](https://github.com/ClickHouse/ClickHouse/pull/31095) ([Amos Bird](https://github.com/amosbird)).
* Support `IF EXISTS` modifier for `RENAME DATABASE`/`TABLE`/`DICTIONARY` query. If this directive is used, one will not get an error if the DATABASE/TABLE/DICTIONARY to be renamed doesn't exist. [#31081](https://github.com/ClickHouse/ClickHouse/pull/31081) ([victorgao](https://github.com/kafka1991)).
* Cancel vertical merges when partition is dropped. This is a follow-up of https://github.com/ClickHouse/ClickHouse/pull/25684 and https://github.com/ClickHouse/ClickHouse/pull/30996. [#31057](https://github.com/ClickHouse/ClickHouse/pull/31057) ([Amos Bird](https://github.com/amosbird)).
* The local session inside a Clickhouse dictionary source won't send its events to the session log anymore. This fixes a possible deadlock (tsan alert) on shutdown. Also this PR fixes flaky `test_dictionaries_dependency_xml/`. [#31013](https://github.com/ClickHouse/ClickHouse/pull/31013) ([Vitaly Baranov](https://github.com/vitlibar)).
* Less locking in ALTER command. [#31010](https://github.com/ClickHouse/ClickHouse/pull/31010) ([Amos Bird](https://github.com/amosbird)).
* Fix `--verbose` option in clickhouse-local interactive mode and allow logging into file. [#30881](https://github.com/ClickHouse/ClickHouse/pull/30881) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Added `\l`, `\d`, `\c` commands in `clickhouse-client` like in MySQL and PostgreSQL. [#30876](https://github.com/ClickHouse/ClickHouse/pull/30876) ([Pavel Medvedev](https://github.com/pmed)).
* For clickhouse-local or clickhouse-client: if there is `--interactive` option with `--query` or `--queries-file`, then first execute them like in non-interactive and then start interactive mode. [#30851](https://github.com/ClickHouse/ClickHouse/pull/30851) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix possible "The local set of parts of X doesn't look like the set of parts in ZooKeeper" error (if DROP fails during removing znodes from zookeeper). [#30826](https://github.com/ClickHouse/ClickHouse/pull/30826) ([Azat Khuzhin](https://github.com/azat)).
* Avro format works against Kafka. Setting `output_format_avro_rows_in_file` added. [#30351](https://github.com/ClickHouse/ClickHouse/pull/30351) ([Ilya Golshtein](https://github.com/ilejn)).
* Allow to specify one or any number of PostgreSQL schemas for one `MaterializedPostgreSQL` database. Closes [#28901](https://github.com/ClickHouse/ClickHouse/issues/28901). Closes [#29324](https://github.com/ClickHouse/ClickHouse/issues/29324). [#28933](https://github.com/ClickHouse/ClickHouse/pull/28933) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Replaced default ports for clickhouse-keeper internal communication from 44444 to 9234. Fixes [#30879](https://github.com/ClickHouse/ClickHouse/issues/30879). [#31799](https://github.com/ClickHouse/ClickHouse/pull/31799) ([alesapin](https://github.com/alesapin)).
* Implement function transform with Decimal arguments. [#31839](https://github.com/ClickHouse/ClickHouse/pull/31839) ([李帅](https://github.com/loneylee)).
* Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix possible assert in `hdfs` table function/engine, add test. [#31036](https://github.com/ClickHouse/ClickHouse/pull/31036) ([Kruglov Pavel](https://github.com/Avogar)).
#### Bug Fixes
* Fix group by / order by / limit by aliases with positional arguments enabled. Closes [#31173](https://github.com/ClickHouse/ClickHouse/issues/31173). [#31741](https://github.com/ClickHouse/ClickHouse/pull/31741) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix usage of `Buffer` table engine with type `Map`. Fixes [#30546](https://github.com/ClickHouse/ClickHouse/issues/30546). [#31742](https://github.com/ClickHouse/ClickHouse/pull/31742) ([Anton Popov](https://github.com/CurtizJ)).
* Fix reading from `MergeTree` tables with enabled `use_uncompressed_cache`. [#31826](https://github.com/ClickHouse/ClickHouse/pull/31826) ([Anton Popov](https://github.com/CurtizJ)).
* Fixed the behavior when mutations that have nothing to do are stuck (with enabled setting `empty_result_for_aggregation_by_empty_set`). [#32358](https://github.com/ClickHouse/ClickHouse/pull/32358) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
* Fix skipping columns while writing protobuf. This PR fixes [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160), see the comment [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160)#issuecomment-980595318. [#31988](https://github.com/ClickHouse/ClickHouse/pull/31988) ([Vitaly Baranov](https://github.com/vitlibar)).
* Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)).
* Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)).
* Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)).
* Memory amount was incorrectly estimated when ClickHouse is run in containers with cgroup limits. [#31157](https://github.com/ClickHouse/ClickHouse/pull/31157) ([Pavel Medvedev](https://github.com/pmed)).
* Fix `ALTER ... MATERIALIZE COLUMN ...` queries in case when data type of default expression is not equal to the data type of column. [#32348](https://github.com/ClickHouse/ClickHouse/pull/32348) ([Anton Popov](https://github.com/CurtizJ)).
* Fixed crash with SIGFPE in aggregate function `avgWeighted` with `Decimal` argument. Fixes [#32053](https://github.com/ClickHouse/ClickHouse/issues/32053). [#32303](https://github.com/ClickHouse/ClickHouse/pull/32303) ([tavplubix](https://github.com/tavplubix)).
* Server might fail to start with `Cannot attach 1 tables due to cyclic dependencies` error if `Dictionary` table looks at XML-dictionary with the same name, it's fixed. Fixes [#31315](https://github.com/ClickHouse/ClickHouse/issues/31315). [#32288](https://github.com/ClickHouse/ClickHouse/pull/32288) ([tavplubix](https://github.com/tavplubix)).
* Fix parsing error while NaN deserializing for `Nullable(Float)` for `Quoted` escaping rule. [#32190](https://github.com/ClickHouse/ClickHouse/pull/32190) ([Kruglov Pavel](https://github.com/Avogar)).
* XML dictionaries: identifiers, used in table create query, can be qualified to `default_database` during upgrade to newer version. Closes [#31963](https://github.com/ClickHouse/ClickHouse/issues/31963). [#32187](https://github.com/ClickHouse/ClickHouse/pull/32187) ([Maksim Kita](https://github.com/kitaisreal)).
* Number of active replicas might be determined incorrectly when inserting with quorum if setting `replicated_can_become_leader` is disabled on some replicas. It's fixed. [#32157](https://github.com/ClickHouse/ClickHouse/pull/32157) ([tavplubix](https://github.com/tavplubix)).
* Dictionaries: fix cases when `{condition}` does not work for custom database queries. [#32117](https://github.com/ClickHouse/ClickHouse/pull/32117) ([Maksim Kita](https://github.com/kitaisreal)).
* Fix `CAST` from `Nullable` with `cast_keep_nullable` (`PARAMETER_OUT_OF_BOUND` error before for i.e. `toUInt32OrDefault(toNullable(toUInt32(1)))`). [#32080](https://github.com/ClickHouse/ClickHouse/pull/32080) ([Azat Khuzhin](https://github.com/azat)).
* Fix CREATE TABLE of Join Storage in some obscure cases. Close [#31680](https://github.com/ClickHouse/ClickHouse/issues/31680). [#32066](https://github.com/ClickHouse/ClickHouse/pull/32066) ([SuperDJY](https://github.com/cmsxbc)).
* Fixed `Directory ... already exists and is not empty` error when detaching part. [#32063](https://github.com/ClickHouse/ClickHouse/pull/32063) ([tavplubix](https://github.com/tavplubix)).
* `MaterializedMySQL` (experimental feature): Fix misinterpretation of `DECIMAL` data from MySQL. [#31990](https://github.com/ClickHouse/ClickHouse/pull/31990) ([Håvard Kvålen](https://github.com/havardk)).
* `FileLog` (experimental feature) engine unnesessary created meta data directory when create table failed. Fix [#31962](https://github.com/ClickHouse/ClickHouse/issues/31962). [#31967](https://github.com/ClickHouse/ClickHouse/pull/31967) ([flynn](https://github.com/ucasfl)).
* Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([tavplubix](https://github.com/tavplubix)).
* Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)).
* Change configuration path from `keeper_server.session_timeout_ms` to `keeper_server.coordination_settings.session_timeout_ms` when constructing a `KeeperTCPHandler`. Same with `operation_timeout`. [#31859](https://github.com/ClickHouse/ClickHouse/pull/31859) ([JackyWoo](https://github.com/JackyWoo)).
* Fix invalid cast of Nullable type when nullable primary key is used. (Nullable primary key is a discouraged feature - please do not use). This fixes [#31075](https://github.com/ClickHouse/ClickHouse/issues/31075). [#31823](https://github.com/ClickHouse/ClickHouse/pull/31823) ([Amos Bird](https://github.com/amosbird)).
* Fix crash in recursive UDF in SQL. Closes [#30856](https://github.com/ClickHouse/ClickHouse/issues/30856). [#31820](https://github.com/ClickHouse/ClickHouse/pull/31820) ([Maksim Kita](https://github.com/kitaisreal)).
* Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)).
* Fix crash with empty result of ODBC query (with some ODBC drivers). Closes [#31465](https://github.com/ClickHouse/ClickHouse/issues/31465). [#31766](https://github.com/ClickHouse/ClickHouse/pull/31766) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix disabling query profiler (In case of `query_profiler_real_time_period_ns>0`/`query_profiler_cpu_time_period_ns>0` query profiler can stayed enabled even after query finished). [#31740](https://github.com/ClickHouse/ClickHouse/pull/31740) ([Azat Khuzhin](https://github.com/azat)).
* Fixed rare segfault on concurrent `ATTACH PARTITION` queries. [#31738](https://github.com/ClickHouse/ClickHouse/pull/31738) ([tavplubix](https://github.com/tavplubix)).
* Fix race in JSONEachRowWithProgress output format when data and lines with progress are mixed in output. [#31736](https://github.com/ClickHouse/ClickHouse/pull/31736) ([Kruglov Pavel](https://github.com/Avogar)).
* Fixed `there are no such cluster here` error on execution of `ON CLUSTER` query if specified cluster name is name of `Replicated` database. [#31723](https://github.com/ClickHouse/ClickHouse/pull/31723) ([tavplubix](https://github.com/tavplubix)).
* Fix exception on some of the applications of `decrypt` function on Nullable columns. This closes [#31662](https://github.com/ClickHouse/ClickHouse/issues/31662). This closes [#31426](https://github.com/ClickHouse/ClickHouse/issues/31426). [#31707](https://github.com/ClickHouse/ClickHouse/pull/31707) ([alexey-milovidov](https://github.com/alexey-milovidov)).
* Fixed function ngrams when string contains UTF-8 characters. [#31706](https://github.com/ClickHouse/ClickHouse/pull/31706) ([yandd](https://github.com/yandd)).
* Settings `input_format_allow_errors_num` and `input_format_allow_errors_ratio` did not work for parsing of domain types, such as `IPv4`, it's fixed. Fixes [#31686](https://github.com/ClickHouse/ClickHouse/issues/31686). [#31697](https://github.com/ClickHouse/ClickHouse/pull/31697) ([tavplubix](https://github.com/tavplubix)).
* Fixed null pointer exception in `MATERIALIZE COLUMN`. [#31679](https://github.com/ClickHouse/ClickHouse/pull/31679) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* `RENAME TABLE` query worked incorrectly on attempt to rename an DDL dictionary in `Ordinary` database, it's fixed. [#31638](https://github.com/ClickHouse/ClickHouse/pull/31638) ([tavplubix](https://github.com/tavplubix)).
* Implement `sparkbar` aggregate function as it was intended, see: [#26175](https://github.com/ClickHouse/ClickHouse/issues/26175)#issuecomment-960353867, [comment](https://github.com/ClickHouse/ClickHouse/issues/26175#issuecomment-961155065). [#31624](https://github.com/ClickHouse/ClickHouse/pull/31624) ([小路](https://github.com/nicelulu)).
* Fix invalid generated JSON when only column names contain invalid UTF-8 sequences. [#31534](https://github.com/ClickHouse/ClickHouse/pull/31534) ([Kevin Michel](https://github.com/kmichel-aiven)).
* Disable `partial_merge_join_left_table_buffer_bytes` before bug in this optimization is fixed. See [#31009](https://github.com/ClickHouse/ClickHouse/issues/31009)). Remove redundant option `partial_merge_join_optimizations`. [#31528](https://github.com/ClickHouse/ClickHouse/pull/31528) ([Vladimir C](https://github.com/vdimir)).
* Fix progress for short `INSERT SELECT` queries. [#31510](https://github.com/ClickHouse/ClickHouse/pull/31510) ([Azat Khuzhin](https://github.com/azat)).
* Fix wrong behavior with group by and positional arguments. Closes [#31280](https://github.com/ClickHouse/ClickHouse/issues/31280)#issuecomment-968696186. [#31420](https://github.com/ClickHouse/ClickHouse/pull/31420) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)).
* Remove `notLike` function from index analysis, because it was wrong. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)).
* Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)).
* Rewrite right distributed table in local join. solves [#25809](https://github.com/ClickHouse/ClickHouse/issues/25809). [#31105](https://github.com/ClickHouse/ClickHouse/pull/31105) ([abel-cheng](https://github.com/abel-cheng)).
* Fix `Merge` table with aliases and where (it did not work before at all). Closes [#28802](https://github.com/ClickHouse/ClickHouse/issues/28802). [#31044](https://github.com/ClickHouse/ClickHouse/pull/31044) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix JSON_VALUE/JSON_QUERY with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Using `formatRow` function with not row-oriented formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix bug which broke select queries if they happened after dropping materialized view. Found in [#30691](https://github.com/ClickHouse/ClickHouse/issues/30691). [#30997](https://github.com/ClickHouse/ClickHouse/pull/30997) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Skip `max_partition_size_to_drop check` in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)).
* Fix some corner cases with `INTERSECT` and `EXCEPT` operators. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)).
#### Build/Testing/Packaging Improvement
* Fix incorrect filtering result on non-x86 builds. This closes [#31417](https://github.com/ClickHouse/ClickHouse/issues/31417). This closes [#31524](https://github.com/ClickHouse/ClickHouse/issues/31524). [#31574](https://github.com/ClickHouse/ClickHouse/pull/31574) ([alexey-milovidov](https://github.com/alexey-milovidov)).
* Make ClickHouse build fully reproducible (byte identical on different machines). This closes [#22113](https://github.com/ClickHouse/ClickHouse/issues/22113). [#31899](https://github.com/ClickHouse/ClickHouse/pull/31899) ([alexey-milovidov](https://github.com/alexey-milovidov)). Remove filesystem path to the build directory from binaries to enable reproducible builds. This needed for [#22113](https://github.com/ClickHouse/ClickHouse/issues/22113). [#31838](https://github.com/ClickHouse/ClickHouse/pull/31838) ([alexey-milovidov](https://github.com/alexey-milovidov)).
* Use our own CMakeLists for `zlib-ng`, `cassandra`, `mariadb-connector-c` and `xz`, `re2`, `sentry`, `gsasl`, `arrow`, `protobuf`. This is needed for [#20151](https://github.com/ClickHouse/ClickHouse/issues/20151). Part of [#9226](https://github.com/ClickHouse/ClickHouse/issues/9226). A small step towards removal of annoying trash from the build system. [#30599](https://github.com/ClickHouse/ClickHouse/pull/30599) ([alexey-milovidov](https://github.com/alexey-milovidov)).
* Hermetic builds: use fixed version of libc and make sure that no source or binary files from the host OS are using during build. This closes [#27133](https://github.com/ClickHouse/ClickHouse/issues/27133). This closes [#21435](https://github.com/ClickHouse/ClickHouse/issues/21435). This closes [#30462](https://github.com/ClickHouse/ClickHouse/issues/30462). [#30011](https://github.com/ClickHouse/ClickHouse/pull/30011) ([alexey-milovidov](https://github.com/alexey-milovidov)).
* Adding function `getFuzzerData()` to easily fuzz particular functions. This closes [#23227](https://github.com/ClickHouse/ClickHouse/issues/23227). [#27526](https://github.com/ClickHouse/ClickHouse/pull/27526) ([Alexey Boykov](https://github.com/mathalex)).
* More correct setting up capabilities inside Docker. [#31802](https://github.com/ClickHouse/ClickHouse/pull/31802) ([Constantine Peresypkin](https://github.com/pkit)).
* Enable clang `-fstrict-vtable-pointers`, `-fwhole-program-vtables` compile options. [#20151](https://github.com/ClickHouse/ClickHouse/pull/20151) ([Maksim Kita](https://github.com/kitaisreal)).
* Avoid downloading toolchain tarballs for cross-compiling for FreeBSD. [#31672](https://github.com/ClickHouse/ClickHouse/pull/31672) ([alexey-milovidov](https://github.com/alexey-milovidov)).
* Initial support for risc-v. See development/build-cross-riscv for quirks and build command that was tested. [#31309](https://github.com/ClickHouse/ClickHouse/pull/31309) ([Vladimir Smirnov](https://github.com/Civil)).
* Support compile in arm machine with parameter "-DENABLE_TESTS=OFF". [#31007](https://github.com/ClickHouse/ClickHouse/pull/31007) ([zhanghuajie](https://github.com/zhanghuajieHIT)).
### ClickHouse release v21.11, 2021-11-09 ### ClickHouse release v21.11, 2021-11-09
#### Backward Incompatible Change #### Backward Incompatible Change

3
CMakeLists.txt
View File

@ -447,7 +447,7 @@ if (MAKE_STATIC_LIBRARIES)
# It's disabled for ARM because otherwise ClickHouse cannot run on Android. # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie") set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fno-pie") set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fno-pie")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no-pie") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -no-pie -Wl,-no-pie")
endif () endif ()
else () else ()
set (CMAKE_POSITION_INDEPENDENT_CODE ON) set (CMAKE_POSITION_INDEPENDENT_CODE ON)
@ -508,6 +508,7 @@ include (cmake/find/hdfs3.cmake) # uses protobuf
include (cmake/find/poco.cmake) include (cmake/find/poco.cmake)
include (cmake/find/curl.cmake) include (cmake/find/curl.cmake)
include (cmake/find/s3.cmake) include (cmake/find/s3.cmake)
include (cmake/find/blob_storage.cmake)
include (cmake/find/base64.cmake) include (cmake/find/base64.cmake)
include (cmake/find/parquet.cmake) include (cmake/find/parquet.cmake)
include (cmake/find/simdjson.cmake) include (cmake/find/simdjson.cmake)

8
CONTRIBUTING.md
View File

@ -2,7 +2,13 @@
ClickHouse is an open project, and you can contribute to it in many ways. You can help with ideas, code, or documentation. We appreciate any efforts that help us to make the project better. ClickHouse is an open project, and you can contribute to it in many ways. You can help with ideas, code, or documentation. We appreciate any efforts that help us to make the project better.
Thank you. Thank you!
## Legal Info
When you open your first pull-request to ClickHouse repo, a bot will invite you to accept ClickHouse Individual CLA (Contributor License Agreement). It is a simple few click process. For subsequent pull-requests the bot will check if you have already signed it and won't bother you again.
Optionally, to make contributions even more tight legally, your employer as a legal entity may want to sign a ClickHouse Corporate CLA with ClickHouse, Inc. If you're interested to do so, contact us at [legal@clickhouse.com](mailto:legal@clickhouse.com).
## Technical Info ## Technical Info

24
base/base/CMakeLists.txt
View File

@ -24,8 +24,6 @@ set (SRCS
if (ENABLE_REPLXX) if (ENABLE_REPLXX)
list (APPEND SRCS ReplxxLineReader.cpp) list (APPEND SRCS ReplxxLineReader.cpp)
elseif (ENABLE_READLINE)
list (APPEND SRCS ReadlineLineReader.cpp)
endif () endif ()
if (USE_DEBUG_HELPERS) if (USE_DEBUG_HELPERS)
@ -52,28 +50,6 @@ if (OS_DARWIN AND NOT MAKE_STATIC_LIBRARIES)
target_link_libraries(common PUBLIC -Wl,-U,_inside_main) target_link_libraries(common PUBLIC -Wl,-U,_inside_main)
endif() endif()
# Allow explicit fallback to readline
if (NOT ENABLE_REPLXX AND ENABLE_READLINE)
message (STATUS "Attempt to fallback to readline explicitly")
set (READLINE_PATHS "/usr/local/opt/readline/lib")
# First try find custom lib for macos users (default lib without history support)
find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH)
if (NOT READLINE_LIB)
find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS})
endif ()
set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include")
find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS} NO_DEFAULT_PATH)
if (NOT READLINE_INCLUDE_DIR)
find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
endif ()
if (READLINE_INCLUDE_DIR AND READLINE_LIB)
target_link_libraries(common PUBLIC ${READLINE_LIB})
target_compile_definitions(common PUBLIC USE_READLINE=1)
message (STATUS "Using readline: ${READLINE_INCLUDE_DIR} : ${READLINE_LIB}")
endif ()
endif ()
target_link_libraries (common target_link_libraries (common
PUBLIC PUBLIC
${CITYHASH_LIBRARIES} ${CITYHASH_LIBRARIES}

37
base/base/LineReader.cpp
View File

@ -10,16 +10,6 @@
#include <sys/types.h> #include <sys/types.h>
#ifdef OS_LINUX
/// We can detect if code is linked with one or another readline variants or open the library dynamically.
# include <dlfcn.h>
extern "C"
{
char * readline(const char *) __attribute__((__weak__));
char * (*readline_ptr)(const char *) = readline;
}
#endif
#ifdef HAS_RESERVED_IDENTIFIER #ifdef HAS_RESERVED_IDENTIFIER
#pragma clang diagnostic ignored "-Wreserved-identifier" #pragma clang diagnostic ignored "-Wreserved-identifier"
#endif #endif
@ -152,33 +142,6 @@ LineReader::InputStatus LineReader::readOneLine(const String & prompt)
{ {
input.clear(); input.clear();
#ifdef OS_LINUX
if (!readline_ptr)
{
for (const auto * name : {"libreadline.so", "libreadline.so.0", "libeditline.so", "libeditline.so.0"})
{
void * dl_handle = dlopen(name, RTLD_LAZY);
if (dl_handle)
{
readline_ptr = reinterpret_cast<char * (*)(const char *)>(dlsym(dl_handle, "readline"));
if (readline_ptr)
{
break;
}
}
}
}
/// Minimal support for readline
if (readline_ptr)
{
char * line_read = (*readline_ptr)(prompt.c_str());
if (!line_read)
return ABORT;
input = line_read;
}
else
#endif
{ {
std::cout << prompt; std::cout << prompt;
std::getline(std::cin, input); std::getline(std::cin, input);

187
base/base/ReadlineLineReader.cpp
View File

@ -1,187 +0,0 @@
#include <base/ReadlineLineReader.h>
#include <base/errnoToString.h>
#include <base/scope_guard.h>
#include <errno.h>
#include <signal.h>
#include <string.h>
#include <unistd.h>
#include <iostream>
namespace
{
/// Trim ending whitespace inplace
void trim(String & s)
{
s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
}
}
static const LineReader::Suggest * suggest;
/// Points to current word to suggest.
static LineReader::Suggest::Words::const_iterator pos;
/// Points after the last possible match.
static LineReader::Suggest::Words::const_iterator end;
/// Set iterators to the matched range of words if any.
static void findRange(const char * prefix, size_t prefix_length)
{
std::string prefix_str(prefix);
if (auto completions = suggest->getCompletions(prefix_str, prefix_length))
std::tie(pos, end) = *completions;
}
/// Iterates through matched range.
static char * nextMatch()
{
if (pos >= end)
return nullptr;
/// readline will free memory by itself.
char * word = strdup(pos->c_str());
++pos;
return word;
}
static char * generate(const char * text, int state)
{
if (!suggest->ready)
return nullptr;
if (state == 0)
findRange(text, strlen(text));
/// Do not append whitespace after word. For unknown reason, rl_completion_append_character = '\0' does not work.
rl_completion_suppress_append = 1;
return nextMatch();
};
ReadlineLineReader::ReadlineLineReader(
const Suggest & suggest_, const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_)
: LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_))
{
suggest = &suggest_;
if (!history_file_path.empty())
{
int res = read_history(history_file_path.c_str());
if (res)
std::cerr << "Cannot read history from file " + history_file_path + ": "+ errnoToString(errno) << std::endl;
}
/// Added '.' to the default list. Because it is used to separate database and table.
rl_basic_word_break_characters = word_break_characters;
/// Not append whitespace after single suggestion. Because whitespace after function name is meaningless.
rl_completion_append_character = '\0';
rl_completion_entry_function = generate;
/// Install Ctrl+C signal handler that will be used in interactive mode.
if (rl_initialize())
throw std::runtime_error("Cannot initialize readline");
auto clear_prompt_or_exit = [](int)
{
/// This is signal safe.
ssize_t res = write(STDOUT_FILENO, "\n", 1);
/// Allow to quit client while query is in progress by pressing Ctrl+C twice.
/// (First press to Ctrl+C will try to cancel query by InterruptListener).
if (res == 1 && rl_line_buffer[0] && !RL_ISSTATE(RL_STATE_DONE))
{
rl_replace_line("", 0);
if (rl_forced_update_display())
_exit(0);
}
else
{
/// A little dirty, but we struggle to find better way to correctly
/// force readline to exit after returning from the signal handler.
_exit(0);
}
};
if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR)
throw std::runtime_error(std::string("Cannot set signal handler for readline: ") + errnoToString(errno));
rl_variable_bind("completion-ignore-case", "on");
// TODO: it doesn't work
// history_write_timestamps = 1;
}
ReadlineLineReader::~ReadlineLineReader()
{
}
LineReader::InputStatus ReadlineLineReader::readOneLine(const String & prompt)
{
input.clear();
const char* cinput = readline(prompt.c_str());
if (cinput == nullptr)
return (errno != EAGAIN) ? ABORT : RESET_LINE;
input = cinput;
trim(input);
return INPUT_LINE;
}
void ReadlineLineReader::addToHistory(const String & line)
{
add_history(line.c_str());
// Flush changes to the disk
// NOTE readline builds a buffer of all the lines to write, and write them in one syscall.
// Thus there is no need to lock the history file here.
write_history(history_file_path.c_str());
}
#if RL_VERSION_MAJOR >= 7
#define BRACK_PASTE_PREF "\033[200~"
#define BRACK_PASTE_SUFF "\033[201~"
#define BRACK_PASTE_LAST '~'
#define BRACK_PASTE_SLEN 6
/// This handler bypasses some unused macro/event checkings and remove trailing newlines before insertion.
static int clickhouse_rl_bracketed_paste_begin(int /* count */, int /* key */)
{
std::string buf;
buf.reserve(128);
RL_SETSTATE(RL_STATE_MOREINPUT);
SCOPE_EXIT(RL_UNSETSTATE(RL_STATE_MOREINPUT));
int c;
while ((c = rl_read_key()) >= 0)
{
if (c == '\r')
c = '\n';
buf.push_back(c);
if (buf.size() >= BRACK_PASTE_SLEN && c == BRACK_PASTE_LAST && buf.substr(buf.size() - BRACK_PASTE_SLEN) == BRACK_PASTE_SUFF)
{
buf.resize(buf.size() - BRACK_PASTE_SLEN);
break;
}
}
trim(buf);
return static_cast<size_t>(rl_insert_text(buf.c_str())) == buf.size() ? 0 : 1;
}
#endif
void ReadlineLineReader::enableBracketedPaste()
{
#if RL_VERSION_MAJOR >= 7
rl_variable_bind("enable-bracketed-paste", "on");
/// Use our bracketed paste handler to get better user experience. See comments above.
rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin);
#endif
};

19
base/base/ReadlineLineReader.h
View File

@ -1,19 +0,0 @@
#pragma once
#include "LineReader.h"
#include <readline/readline.h>
#include <readline/history.h>
class ReadlineLineReader : public LineReader
{
public:
ReadlineLineReader(const Suggest & suggest, const String & history_file_path, bool multiline, Patterns extenders_, Patterns delimiters_);
~ReadlineLineReader() override;
void enableBracketedPaste() override;
private:
InputStatus readOneLine(const String & prompt) override;
void addToHistory(const String & line) override;
};

28
cmake/find/blob_storage.cmake Normal file
View File

@ -0,0 +1,28 @@
option(USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY
"Set to FALSE to use system Azure SDK instead of bundled (OFF currently not implemented)"
ON)
if (USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
set(USE_AZURE_BLOB_STORAGE 1)
set(AZURE_BLOB_STORAGE_LIBRARY azure_sdk)
endif()
if ((NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/azure/sdk"
OR NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/azure/cmake-modules")
AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
message (WARNING "submodule contrib/azure is missing. to fix try run: \n git submodule update --init")
set(USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY OFF)
set(USE_AZURE_BLOB_STORAGE 0)
endif ()
if (NOT USE_INTERNAL_SSL_LIBRARY AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
message (FATAL_ERROR "Currently Blob Storage support can be built only with internal SSL library")
endif()
if (NOT USE_INTERNAL_CURL AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
message (FATAL_ERROR "Currently Blob Storage support can be built only with internal curl library")
endif()
if (USE_AZURE_BLOB_STORAGE)
message (STATUS "Using Azure Blob Storage - ${USE_AZURE_BLOB_STORAGE}")
endif()

3
cmake/tools.cmake
View File

@ -91,6 +91,9 @@ endif ()
if (LINKER_NAME) if (LINKER_NAME)
if (COMPILER_CLANG AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 12.0.0 OR CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 12.0.0)) if (COMPILER_CLANG AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 12.0.0 OR CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 12.0.0))
find_program (LLD_PATH NAMES ${LINKER_NAME}) find_program (LLD_PATH NAMES ${LINKER_NAME})
if (NOT LLD_PATH)
message (FATAL_ERROR "Using linker ${LINKER_NAME} but can't find its path.")
endif ()
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LLD_PATH}") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LLD_PATH}")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_PATH}") set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_PATH}")
else () else ()

4
contrib/CMakeLists.txt vendored
View File

@ -249,6 +249,10 @@ endif()
# - sentry-native # - sentry-native
add_subdirectory (curl-cmake) add_subdirectory (curl-cmake)
if (USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY)
add_subdirectory(azure-cmake)
endif()
if (USE_SENTRY) if (USE_SENTRY)
add_subdirectory (sentry-native-cmake) add_subdirectory (sentry-native-cmake)
endif() endif()

1
contrib/azure vendored Submodule

@ -0,0 +1 @@
Subproject commit ac4b763d4ca40122275f1497cbdc5451337461d9

71
contrib/azure-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,71 @@
set(AZURE_DIR "${ClickHouse_SOURCE_DIR}/contrib/azure")
set(AZURE_SDK_LIBRARY_DIR "${AZURE_DIR}/sdk")
file(GLOB AZURE_SDK_CORE_SRC
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/cryptography/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/curl/*.hpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/curl/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/winhttp/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/io/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/private/*.hpp"
)
file(GLOB AZURE_SDK_IDENTITY_SRC
"${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/src/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/src/private/*.hpp"
)
file(GLOB AZURE_SDK_STORAGE_COMMON_SRC
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/src/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/src/private/*.cpp"
)
file(GLOB AZURE_SDK_STORAGE_BLOBS_SRC
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/*.cpp"
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/private/*.hpp"
)
file(GLOB AZURE_SDK_UNIFIED_SRC
${AZURE_SDK_CORE_SRC}
${AZURE_SDK_IDENTITY_SRC}
${AZURE_SDK_STORAGE_COMMON_SRC}
${AZURE_SDK_STORAGE_BLOBS_SRC}
)
set(AZURE_SDK_INCLUDES
"${AZURE_SDK_LIBRARY_DIR}/core/azure-core/inc/"
"${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/inc/"
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/inc/"
"${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/inc/"
)
include("${AZURE_DIR}/cmake-modules/AzureTransportAdapters.cmake")
add_library(azure_sdk ${AZURE_SDK_UNIFIED_SRC})
if (COMPILER_CLANG)
target_compile_options(azure_sdk PUBLIC
-Wno-deprecated-copy-dtor
-Wno-extra-semi
-Wno-suggest-destructor-override
-Wno-inconsistent-missing-destructor-override
-Wno-error=unknown-warning-option
-Wno-reserved-identifier
)
endif()
# Originally, on Windows azure-core is built with bcrypt and crypt32 by default
if (OPENSSL_FOUND)
target_link_libraries(azure_sdk PRIVATE ${OPENSSL_LIBRARIES})
endif()
# Originally, on Windows azure-core is built with winhttp by default
if (CURL_FOUND)
target_link_libraries(azure_sdk PRIVATE ${CURL_LIBRARY})
endif()
target_link_libraries(azure_sdk PRIVATE ${LIBXML2_LIBRARIES})
target_include_directories(azure_sdk PUBLIC ${AZURE_SDK_INCLUDES})

1
contrib/boringssl-cmake/CMakeLists.txt
View File

@ -639,6 +639,7 @@ add_library(
"${BORINGSSL_SOURCE_DIR}/decrepit/ssl/ssl_decrepit.c" "${BORINGSSL_SOURCE_DIR}/decrepit/ssl/ssl_decrepit.c"
"${BORINGSSL_SOURCE_DIR}/decrepit/cfb/cfb.c" "${BORINGSSL_SOURCE_DIR}/decrepit/cfb/cfb.c"
"${BORINGSSL_SOURCE_DIR}/decrepit/bio/base64_bio.c"
) )
add_executable( add_executable(

2
contrib/poco vendored

@ -1 +1 @@
Subproject commit 258b9ba6cd245ff88e9346f75c43464c403f329d Subproject commit 520a90e02e3e5cb90afeae1846d161dbc508a6f1

2
contrib/replxx-cmake/CMakeLists.txt
View File

@ -8,7 +8,7 @@ if (NOT ENABLE_REPLXX)
add_library(replxx INTERFACE) add_library(replxx INTERFACE)
target_compile_definitions(replxx INTERFACE USE_REPLXX=0) target_compile_definitions(replxx INTERFACE USE_REPLXX=0)
message (STATUS "Not using replxx (Beware! Runtime fallback to readline is possible!)") message (STATUS "Not using replxx")
return() return()
endif() endif()

4
contrib/unixodbc-cmake/linux_x86_64/private/config.h
View File

@ -202,10 +202,10 @@
#define HAVE_READDIR 1 #define HAVE_READDIR 1
/* Add readline support */ /* Add readline support */
#define HAVE_READLINE 1 /* #undef HAVE_READLINE */
/* Define to 1 if you have the <readline/history.h> header file. */ /* Define to 1 if you have the <readline/history.h> header file. */
#define HAVE_READLINE_HISTORY_H 1 /* #undef HAVE_READLINE_HISTORY_H */
/* Use the scandir lib */ /* Use the scandir lib */
/* #undef HAVE_SCANDIR */ /* #undef HAVE_SCANDIR */

1
docker/packager/binary/Dockerfile
View File

@ -52,7 +52,6 @@ RUN apt-get update \
llvm-${LLVM_VERSION} \ llvm-${LLVM_VERSION} \
llvm-${LLVM_VERSION}-dev \ llvm-${LLVM_VERSION}-dev \
libicu-dev \ libicu-dev \
libreadline-dev \
moreutils \ moreutils \
ninja-build \ ninja-build \
pigz \ pigz \

13
docker/test/fasttest/run.sh
View File

@ -111,19 +111,6 @@ function start_server
fi fi
echo "ClickHouse server pid '$server_pid' started and responded" echo "ClickHouse server pid '$server_pid' started and responded"
echo "
set follow-fork-mode child
handle all noprint
handle SIGSEGV stop print
handle SIGBUS stop print
handle SIGABRT stop print
continue
thread apply all backtrace
continue
" > script.gdb
gdb -batch -command script.gdb -p "$server_pid" &
} }
function clone_root function clone_root

36
docker/test/fuzzer/run-fuzzer.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash #!/bin/bash
# shellcheck disable=SC2086,SC2001,SC2046 # shellcheck disable=SC2086,SC2001,SC2046,SC2030,SC2031
set -eux set -eux
set -o pipefail set -o pipefail
@ -35,7 +35,7 @@ function clone
fi fi
git diff --name-only master HEAD | tee ci-changed-files.txt git diff --name-only master HEAD | tee ci-changed-files.txt
else else
if [ -v COMMIT_SHA ]; then if [ -v SHA_TO_TEST ]; then
git fetch --depth 2 origin "$SHA_TO_TEST" git fetch --depth 2 origin "$SHA_TO_TEST"
git checkout "$SHA_TO_TEST" git checkout "$SHA_TO_TEST"
echo "Checked out nominal SHA $SHA_TO_TEST for master" echo "Checked out nominal SHA $SHA_TO_TEST for master"
@ -155,17 +155,34 @@ function fuzz
kill -0 $server_pid kill -0 $server_pid
# Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog
# and clickhouse-server can do fork-exec, for example, to run some bridge.
# Do not set nostop noprint for all signals, because some it may cause gdb to hang,
# explicitly ignore non-fatal signals that are used by server.
# Number of SIGRTMIN can be determined only in runtime.
RTMIN=$(kill -l SIGRTMIN)
echo " echo "
set follow-fork-mode child set follow-fork-mode parent
handle all noprint handle SIGHUP nostop noprint pass
handle SIGSEGV stop print handle SIGINT nostop noprint pass
handle SIGBUS stop print handle SIGQUIT nostop noprint pass
continue handle SIGPIPE nostop noprint pass
thread apply all backtrace handle SIGTERM nostop noprint pass
handle SIGUSR1 nostop noprint pass
handle SIGUSR2 nostop noprint pass
handle SIG$RTMIN nostop noprint pass
info signals
continue continue
backtrace full
info locals
detach
quit
" > script.gdb " > script.gdb
gdb -batch -command script.gdb -p $server_pid & gdb -batch -command script.gdb -p $server_pid &
sleep 5
# gdb will send SIGSTOP, spend some time loading debug info and then send SIGCONT, wait for it (up to send_timeout, 300s)
time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||:
# Check connectivity after we attach gdb, because it might cause the server # Check connectivity after we attach gdb, because it might cause the server
# to freeze and the fuzzer will fail. # to freeze and the fuzzer will fail.
@ -189,6 +206,7 @@ continue
--receive_data_timeout_ms=10000 \ --receive_data_timeout_ms=10000 \
--stacktrace \ --stacktrace \
--query-fuzzer-runs=1000 \ --query-fuzzer-runs=1000 \
--testmode \
--queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \ --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \
$NEW_TESTS_OPT \ $NEW_TESTS_OPT \
> >(tail -n 100000 > fuzzer.log) \ > >(tail -n 100000 > fuzzer.log) \

1
docker/test/integration/base/Dockerfile
View File

@ -7,7 +7,6 @@ RUN apt-get update \
&& env DEBIAN_FRONTEND=noninteractive apt-get -y install \ && env DEBIAN_FRONTEND=noninteractive apt-get -y install \
tzdata \ tzdata \
python3 \ python3 \
libreadline-dev \
libicu-dev \ libicu-dev \
bsdutils \ bsdutils \
gdb \ gdb \

6
docker/test/integration/runner/Dockerfile
View File

@ -21,7 +21,6 @@ RUN apt-get update \
cgroupfs-mount \ cgroupfs-mount \
python3-pip \ python3-pip \
tzdata \ tzdata \
libreadline-dev \
libicu-dev \ libicu-dev \
bsdutils \ bsdutils \
curl \ curl \
@ -76,7 +75,7 @@ RUN python3 -m pip install \
minio \ minio \
protobuf \ protobuf \
psycopg2-binary==2.8.6 \ psycopg2-binary==2.8.6 \
pymongo \ pymongo==3.11.0 \
pytest \ pytest \
pytest-timeout \ pytest-timeout \
pytest-xdist \ pytest-xdist \
@ -86,7 +85,8 @@ RUN python3 -m pip install \
tzlocal==2.1 \ tzlocal==2.1 \
urllib3 \ urllib3 \
requests-kerberos \ requests-kerberos \
pyhdfs pyhdfs \
azure-storage-blob
COPY modprobe.sh /usr/local/bin/modprobe COPY modprobe.sh /usr/local/bin/modprobe
COPY dockerd-entrypoint.sh /usr/local/bin/ COPY dockerd-entrypoint.sh /usr/local/bin/

13
docker/test/integration/runner/compose/docker_compose_azurite.yml Normal file
View File

@ -0,0 +1,13 @@
version: '2.3'
services:
azurite1:
image: mcr.microsoft.com/azure-storage/azurite
ports:
- "10000:10000"
volumes:
- data1-1:/data1
command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log
volumes:
data1-1:

8
docker/test/integration/runner/compose/docker_compose_mongo.yml
View File

@ -1,7 +1,7 @@
version: '2.3' version: '2.3'
services: services:
mongo1: mongo1:
image: mongo:3.6 image: mongo:5.0
restart: always restart: always
environment: environment:
MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_USERNAME: root
@ -9,3 +9,9 @@ services:
ports: ports:
- ${MONGO_EXTERNAL_PORT}:${MONGO_INTERNAL_PORT} - ${MONGO_EXTERNAL_PORT}:${MONGO_INTERNAL_PORT}
command: --profile=2 --verbose command: --profile=2 --verbose
mongo2:
image: mongo:5.0
restart: always
ports:
- "27018:27017"

85
docker/test/performance-comparison/compare.sh
View File

@ -193,7 +193,7 @@ function run_tests
then then
# Run only explicitly specified tests, if any. # Run only explicitly specified tests, if any.
# shellcheck disable=SC2010 # shellcheck disable=SC2010
test_files=$(ls "$test_prefix" | grep "$CHPC_TEST_GREP" | xargs -I{} -n1 readlink -f "$test_prefix/{}") test_files=($(ls "$test_prefix" | grep "$CHPC_TEST_GREP" | xargs -I{} -n1 readlink -f "$test_prefix/{}"))
elif [ "$PR_TO_TEST" -ne 0 ] \ elif [ "$PR_TO_TEST" -ne 0 ] \
&& [ "$(wc -l < changed-test-definitions.txt)" -gt 0 ] \ && [ "$(wc -l < changed-test-definitions.txt)" -gt 0 ] \
&& [ "$(wc -l < other-changed-files.txt)" -eq 0 ] && [ "$(wc -l < other-changed-files.txt)" -eq 0 ]
@ -201,10 +201,26 @@ function run_tests
# If only the perf tests were changed in the PR, we will run only these # If only the perf tests were changed in the PR, we will run only these
# tests. The lists of changed files are prepared in entrypoint.sh because # tests. The lists of changed files are prepared in entrypoint.sh because
# it has the repository. # it has the repository.
test_files=$(sed "s/tests\/performance/${test_prefix//\//\\/}/" changed-test-definitions.txt) test_files=($(sed "s/tests\/performance/${test_prefix//\//\\/}/" changed-test-definitions.txt))
else else
# The default -- run all tests found in the test dir. # The default -- run all tests found in the test dir.
test_files=$(ls "$test_prefix"/*.xml) test_files=($(ls "$test_prefix"/*.xml))
fi
# We split perf tests into multiple checks to make them faster
if [ -v CHPC_TEST_RUN_BY_HASH_TOTAL ]; then
# filter tests array in bash https://stackoverflow.com/a/40375567
for index in "${!test_files[@]}"; do
# sorry for this, just calculating hash(test_name) % total_tests_group == my_test_group_num
test_hash_result=$(echo test_files[$index] | perl -ne 'use Digest::MD5 qw(md5); print unpack('Q', md5($_)) % $ENV{CHPC_TEST_RUN_BY_HASH_TOTAL} == $ENV{CHPC_TEST_RUN_BY_HASH_NUM};')
# BTW, for some reason when hash(test_name) % total_tests_group != my_test_group_num perl outputs nothing, not zero
if [ "$test_hash_result" != "1" ]; then
# deleting element from array
unset -v 'test_files[$index]'
fi
done
# to have sequential indexes...
test_files=("${test_files[@]}")
fi fi
# For PRs w/o changes in test definitons, test only a subset of queries, # For PRs w/o changes in test definitons, test only a subset of queries,
@ -212,21 +228,26 @@ function run_tests
# already set, keep those values. # already set, keep those values.
# #
# NOTE: too high CHPC_RUNS/CHPC_MAX_QUERIES may hit internal CI timeout. # NOTE: too high CHPC_RUNS/CHPC_MAX_QUERIES may hit internal CI timeout.
if [ "$PR_TO_TEST" -ne 0 ] && [ "$(wc -l < changed-test-definitions.txt)" -eq 0 ] # NOTE: Currently we disabled complete run even for master branch
then #if [ "$PR_TO_TEST" -ne 0 ] && [ "$(wc -l < changed-test-definitions.txt)" -eq 0 ]
CHPC_RUNS=${CHPC_RUNS:-7} #then
CHPC_MAX_QUERIES=${CHPC_MAX_QUERIES:-10} # CHPC_RUNS=${CHPC_RUNS:-7}
else # CHPC_MAX_QUERIES=${CHPC_MAX_QUERIES:-10}
CHPC_RUNS=${CHPC_RUNS:-13} #else
CHPC_MAX_QUERIES=${CHPC_MAX_QUERIES:-0} # CHPC_RUNS=${CHPC_RUNS:-13}
fi # CHPC_MAX_QUERIES=${CHPC_MAX_QUERIES:-0}
#fi
CHPC_RUNS=${CHPC_RUNS:-7}
CHPC_MAX_QUERIES=${CHPC_MAX_QUERIES:-10}
export CHPC_RUNS export CHPC_RUNS
export CHPC_MAX_QUERIES export CHPC_MAX_QUERIES
# Determine which concurrent benchmarks to run. For now, the only test # Determine which concurrent benchmarks to run. For now, the only test
# we run as a concurrent benchmark is 'website'. Run it as benchmark if we # we run as a concurrent benchmark is 'website'. Run it as benchmark if we
# are also going to run it as a normal test. # are also going to run it as a normal test.
for test in $test_files; do echo "$test"; done | sed -n '/website/p' > benchmarks-to-run.txt for test in ${test_files[@]}; do echo "$test"; done | sed -n '/website/p' > benchmarks-to-run.txt
# Delete old report files. # Delete old report files.
for x in {test-times,wall-clock-times}.tsv for x in {test-times,wall-clock-times}.tsv
@ -235,8 +256,8 @@ function run_tests
touch "$x" touch "$x"
done done
# Randomize test order. # Randomize test order. BTW, it's not an array no more.
test_files=$(for f in $test_files; do echo "$f"; done | sort -R) test_files=$(for f in ${test_files[@]}; do echo "$f"; done | sort -R)
# Limit profiling time to 10 minutes, not to run for too long. # Limit profiling time to 10 minutes, not to run for too long.
profile_seconds_left=600 profile_seconds_left=600
@ -261,16 +282,24 @@ function run_tests
# Use awk because bash doesn't support floating point arithmetic. # Use awk because bash doesn't support floating point arithmetic.
profile_seconds=$(awk "BEGIN { print ($profile_seconds_left > 0 ? 10 : 0) }") profile_seconds=$(awk "BEGIN { print ($profile_seconds_left > 0 ? 10 : 0) }")
TIMEFORMAT=$(printf "$test_name\t%%3R\t%%3U\t%%3S\n") (
# The grep is to filter out set -x output and keep only time output. set +x
# The '2>&1 >/dev/null' redirects stderr to stdout, and discards stdout. argv=(
{ \ --host localhost localhost
time "$script_dir/perf.py" --host localhost localhost --port $LEFT_SERVER_PORT $RIGHT_SERVER_PORT \ --port "$LEFT_SERVER_PORT" "$RIGHT_SERVER_PORT"
--runs "$CHPC_RUNS" --max-queries "$CHPC_MAX_QUERIES" \ --runs "$CHPC_RUNS"
--profile-seconds "$profile_seconds" \ --max-queries "$CHPC_MAX_QUERIES"
-- "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" ; \ --profile-seconds "$profile_seconds"
} 2>&1 >/dev/null | tee >(grep -v ^+ >> "wall-clock-times.tsv") \
|| echo "Test $test_name failed with error code $?" >> "$test_name-err.log" "$test"
)
TIMEFORMAT=$(printf "$test_name\t%%3R\t%%3U\t%%3S\n")
# one more subshell to suppress trace output for "set +x"
(
time "$script_dir/perf.py" "${argv[@]}" > "$test_name-raw.tsv" 2> "$test_name-err.log"
) 2>>wall-clock-times.tsv >/dev/null \
|| echo "Test $test_name failed with error code $?" >> "$test_name-err.log"
) 2>/dev/null
profile_seconds_left=$(awk -F' ' \ profile_seconds_left=$(awk -F' ' \
'BEGIN { s = '$profile_seconds_left'; } /^profile-total/ { s -= $2 } END { print s }' \ 'BEGIN { s = '$profile_seconds_left'; } /^profile-total/ { s -= $2 } END { print s }' \
@ -278,8 +307,6 @@ function run_tests
current_test=$((current_test + 1)) current_test=$((current_test + 1))
done done
unset TIMEFORMAT
wait wait
} }
@ -291,7 +318,7 @@ function get_profiles_watchdog
for pid in $(pgrep -f clickhouse) for pid in $(pgrep -f clickhouse)
do do
gdb -p "$pid" --batch --ex "info proc all" --ex "thread apply all bt" --ex quit &> "$pid.gdb.log" & sudo gdb -p "$pid" --batch --ex "info proc all" --ex "thread apply all bt" --ex quit &> "$pid.gdb.log" &
done done
wait wait
@ -518,7 +545,9 @@ unset IFS
# all nodes. # all nodes.
numactl --show numactl --show
numactl --cpunodebind=all --membind=all numactl --show numactl --cpunodebind=all --membind=all numactl --show
numactl --cpunodebind=all --membind=all parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log # Use less jobs to avoid OOM. Some queries can consume 8+ GB of memory.
jobs_count=$(($(grep -c ^processor /proc/cpuinfo) / 3))
numactl --cpunodebind=all --membind=all parallel --jobs $jobs_count --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log
clickhouse-local --query " clickhouse-local --query "
-- Join the metric names back to the metric statistics we've calculated, and make -- Join the metric names back to the metric statistics we've calculated, and make

22
docker/test/performance-comparison/download.sh
View File

@ -16,16 +16,28 @@ right_sha=$4
datasets=${CHPC_DATASETS-"hits1 hits10 hits100 values"} datasets=${CHPC_DATASETS-"hits1 hits10 hits100 values"}
declare -A dataset_paths declare -A dataset_paths
dataset_paths["hits10"]="https://s3.mds.yandex.net/clickhouse-private-datasets/hits_10m_single/partitions/hits_10m_single.tar" if [[ $S3_URL == *"s3.amazonaws.com"* ]]; then
dataset_paths["hits100"]="https://s3.mds.yandex.net/clickhouse-private-datasets/hits_100m_single/partitions/hits_100m_single.tar" dataset_paths["hits10"]="https://clickhouse-private-datasets.s3.amazonaws.com/hits_10m_single/partitions/hits_10m_single.tar"
dataset_paths["hits1"]="https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar" dataset_paths["hits100"]="https://clickhouse-private-datasets.s3.amazonaws.com/hits_100m_single/partitions/hits_100m_single.tar"
dataset_paths["values"]="https://clickhouse-datasets.s3.yandex.net/values_with_expressions/partitions/test_values.tar" dataset_paths["hits1"]="https://clickhouse-datasets.s3.amazonaws.com/hits/partitions/hits_v1.tar"
dataset_paths["values"]="https://clickhouse-datasets.s3.amazonaws.com/values_with_expressions/partitions/test_values.tar"
else
dataset_paths["hits10"]="https://s3.mds.yandex.net/clickhouse-private-datasets/hits_10m_single/partitions/hits_10m_single.tar"
dataset_paths["hits100"]="https://s3.mds.yandex.net/clickhouse-private-datasets/hits_100m_single/partitions/hits_100m_single.tar"
dataset_paths["hits1"]="https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar"
dataset_paths["values"]="https://clickhouse-datasets.s3.yandex.net/values_with_expressions/partitions/test_values.tar"
fi
function download function download
{ {
# Historically there were various paths for the performance test package. # Historically there were various paths for the performance test package.
# Test all of them. # Test all of them.
for path in "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/"{,clickhouse_build_check/}"performance/performance.tgz" declare -a urls_to_try=("https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/performance/performance.tgz"
"https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/clickhouse_build_check/performance/performance.tgz"
)
for path in "${urls_to_try[@]}"
do do
if curl --fail --head "$path" if curl --fail --head "$path"
then then

25
docker/test/performance-comparison/entrypoint.sh
View File

@ -4,6 +4,13 @@ set -ex
CHPC_CHECK_START_TIMESTAMP="$(date +%s)" CHPC_CHECK_START_TIMESTAMP="$(date +%s)"
export CHPC_CHECK_START_TIMESTAMP export CHPC_CHECK_START_TIMESTAMP
S3_URL=${S3_URL:="https://clickhouse-builds.s3.yandex.net"}
COMMON_BUILD_PREFIX="/clickhouse_build_check"
if [[ $S3_URL == *"s3.amazonaws.com"* ]]; then
COMMON_BUILD_PREFIX=""
fi
# Use the packaged repository to find the revision we will compare to. # Use the packaged repository to find the revision we will compare to.
function find_reference_sha function find_reference_sha
{ {
@ -43,7 +50,10 @@ function find_reference_sha
# Historically there were various path for the performance test package, # Historically there were various path for the performance test package,
# test all of them. # test all of them.
unset found unset found
for path in "https://clickhouse-builds.s3.yandex.net/0/$REF_SHA/"{,clickhouse_build_check/}"performance/performance.tgz" declare -a urls_to_try=("https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/performance/performance.tgz"
"https://clickhouse-builds.s3.yandex.net/0/$REF_SHA/clickhouse_build_check/performance/performance.tgz"
)
for path in "${urls_to_try[@]}"
do do
if curl --fail --head "$path" if curl --fail --head "$path"
then then
@ -65,14 +75,11 @@ chmod 777 workspace output
cd workspace cd workspace
# Download the package for the version we are going to test # Download the package for the version we are going to test.
for path in "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/"{,clickhouse_build_check/}"performance/performance.tgz" if curl --fail --head "$S3_URL/$PR_TO_TEST/$SHA_TO_TEST$COMMON_BUILD_PREFIX/performance/performance.tgz"
do then
if curl --fail --head "$path" right_path="$S3_URL/$PR_TO_TEST/$SHA_TO_TEST$COMMON_BUILD_PREFIX/performance/performance.tgz"
then fi
right_path="$path"
fi
done
mkdir right mkdir right
wget -nv -nd -c "$right_path" -O- | tar -C right --strip-components=1 -zxv wget -nv -nd -c "$right_path" -O- | tar -C right --strip-components=1 -zxv

9
docker/test/performance-comparison/perf.py
View File

@ -45,6 +45,7 @@ parser.add_argument('--runs', type=int, default=1, help='Number of query runs pe
parser.add_argument('--max-queries', type=int, default=None, help='Test no more than this number of queries, chosen at random.') parser.add_argument('--max-queries', type=int, default=None, help='Test no more than this number of queries, chosen at random.')
parser.add_argument('--queries-to-run', nargs='*', type=int, default=None, help='Space-separated list of indexes of queries to test.') parser.add_argument('--queries-to-run', nargs='*', type=int, default=None, help='Space-separated list of indexes of queries to test.')
parser.add_argument('--max-query-seconds', type=int, default=15, help='For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.') parser.add_argument('--max-query-seconds', type=int, default=15, help='For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.')
parser.add_argument('--prewarm-max-query-seconds', type=int, default=180, help='For how many seconds at most a prewarm (cold storage) query is allowed to run. The script finishes with error if this time is exceeded.')
parser.add_argument('--profile-seconds', type=int, default=0, help='For how many seconds to profile a query for which the performance has changed.') parser.add_argument('--profile-seconds', type=int, default=0, help='For how many seconds to profile a query for which the performance has changed.')
parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.') parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.')
parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.') parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.')
@ -284,7 +285,7 @@ for query_index in queries_to_run:
# it makes the results unstable. # it makes the results unstable.
res = c.execute(q, query_id = prewarm_id, res = c.execute(q, query_id = prewarm_id,
settings = { settings = {
'max_execution_time': args.max_query_seconds, 'max_execution_time': args.prewarm_max_query_seconds,
'query_profiler_real_time_period_ns': 10000000, 'query_profiler_real_time_period_ns': 10000000,
'memory_profiler_step': '4Mi', 'memory_profiler_step': '4Mi',
}) })
@ -354,11 +355,9 @@ for query_index in queries_to_run:
print(f'query\t{query_index}\t{run_id}\t{conn_index}\t{elapsed}') print(f'query\t{query_index}\t{run_id}\t{conn_index}\t{elapsed}')
if elapsed > args.max_query_seconds: if elapsed > args.max_query_seconds:
# Stop processing pathologically slow queries, to avoid timing out # Do not stop processing pathologically slow queries,
# the entire test task. This shouldn't really happen, so we don't # since this may hide errors in other queries.
# need much handling for this case and can just exit.
print(f'The query no. {query_index} is taking too long to run ({elapsed} s)', file=sys.stderr) print(f'The query no. {query_index} is taking too long to run ({elapsed} s)', file=sys.stderr)
exit(2)
# Be careful with the counter, after this line it's the next iteration # Be careful with the counter, after this line it's the next iteration
# already. # already.

15
docker/test/stateful/run.sh
View File

@ -61,6 +61,7 @@ chmod 777 -R /var/lib/clickhouse
clickhouse-client --query "SHOW DATABASES" clickhouse-client --query "SHOW DATABASES"
clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary" clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
service clickhouse-server restart service clickhouse-server restart
# Wait for server to start accepting connections # Wait for server to start accepting connections
@ -109,15 +110,25 @@ function run_tests()
fi fi
set +e set +e
clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time "${ADDITIONAL_OPTIONS[@]}" \ clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time \
--skip 00168_parallel_processing_on_replicas "${ADDITIONAL_OPTIONS[@]}" \
"$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
clickhouse-test --timeout 1200 --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time \
00168_parallel_processing_on_replicas "${ADDITIONAL_OPTIONS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt
set -e set -e
} }
export -f run_tests export -f run_tests
timeout "$MAX_RUN_TIME" bash -c run_tests ||: timeout "$MAX_RUN_TIME" bash -c run_tests ||:
./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv echo "Files in current directory"
ls -la ./
echo "Files in root directory"
ls -la /
/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||: grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:

1
docker/test/stateless/Dockerfile
View File

@ -49,7 +49,6 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
ENV NUM_TRIES=1 ENV NUM_TRIES=1
ENV MAX_RUN_TIME=0 ENV MAX_RUN_TIME=0
# Download Minio-related binaries # Download Minio-related binaries
RUN wget 'https://dl.min.io/server/minio/release/linux-amd64/minio' \ RUN wget 'https://dl.min.io/server/minio/release/linux-amd64/minio' \
&& chmod +x ./minio \ && chmod +x ./minio \

14
docker/test/stateless/run.sh
View File

@ -96,6 +96,13 @@ function run_tests()
ADDITIONAL_OPTIONS+=('8') ADDITIONAL_OPTIONS+=('8')
fi fi
if [[ -n "$RUN_BY_HASH_NUM" ]] && [[ -n "$RUN_BY_HASH_TOTAL" ]]; then
ADDITIONAL_OPTIONS+=('--run-by-hash-num')
ADDITIONAL_OPTIONS+=("$RUN_BY_HASH_NUM")
ADDITIONAL_OPTIONS+=('--run-by-hash-total')
ADDITIONAL_OPTIONS+=("$RUN_BY_HASH_TOTAL")
fi
set +e set +e
clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \ clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
--test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \ --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
@ -108,7 +115,12 @@ export -f run_tests
timeout "$MAX_RUN_TIME" bash -c run_tests ||: timeout "$MAX_RUN_TIME" bash -c run_tests ||:
./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv echo "Files in current directory"
ls -la ./
echo "Files in root directory"
ls -la /
/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
clickhouse-client -q "system flush logs" ||: clickhouse-client -q "system flush logs" ||:

33
docker/test/stress/run.sh
View File

@ -1,6 +1,7 @@
#!/bin/bash #!/bin/bash
# shellcheck disable=SC2094 # shellcheck disable=SC2094
# shellcheck disable=SC2086 # shellcheck disable=SC2086
# shellcheck disable=SC2024
set -x set -x
@ -127,14 +128,26 @@ function start()
counter=$((counter + 1)) counter=$((counter + 1))
done done
# Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog
# and clickhouse-server can do fork-exec, for example, to run some bridge.
# Do not set nostop noprint for all signals, because some it may cause gdb to hang,
# explicitly ignore non-fatal signals that are used by server.
# Number of SIGRTMIN can be determined only in runtime.
RTMIN=$(kill -l SIGRTMIN)
echo " echo "
set follow-fork-mode child set follow-fork-mode parent
handle all noprint handle SIGHUP nostop noprint pass
handle SIGSEGV stop print handle SIGINT nostop noprint pass
handle SIGBUS stop print handle SIGQUIT nostop noprint pass
handle SIGABRT stop print handle SIGPIPE nostop noprint pass
handle SIGTERM nostop noprint pass
handle SIGUSR1 nostop noprint pass
handle SIGUSR2 nostop noprint pass
handle SIG$RTMIN nostop noprint pass
info signals
continue continue
thread apply all backtrace backtrace full
info locals
detach detach
quit quit
" > script.gdb " > script.gdb
@ -142,7 +155,10 @@ quit
# FIXME Hung check may work incorrectly because of attached gdb # FIXME Hung check may work incorrectly because of attached gdb
# 1. False positives are possible # 1. False positives are possible
# 2. We cannot attach another gdb to get stacktraces if some queries hung # 2. We cannot attach another gdb to get stacktraces if some queries hung
gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" >> /test_output/gdb.log & gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log &
sleep 5
# gdb will send SIGSTOP, spend some time loading debug info and then send SIGCONT, wait for it (up to send_timeout, 300s)
time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||:
} }
configure configure
@ -213,6 +229,9 @@ zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.log* > /dev/n
zgrep -Fa "########################################" /test_output/* > /dev/null \ zgrep -Fa "########################################" /test_output/* > /dev/null \
&& echo -e 'Killed by signal (output files)\tFAIL' >> /test_output/test_results.tsv && echo -e 'Killed by signal (output files)\tFAIL' >> /test_output/test_results.tsv
zgrep -Fa " received signal " /test_output/gdb.log > /dev/null \
&& echo -e 'Found signal in gdb.log\tFAIL' >> /test_output/test_results.tsv
# Put logs into /test_output/ # Put logs into /test_output/
for log_file in /var/log/clickhouse-server/clickhouse-server.log* for log_file in /var/log/clickhouse-server/clickhouse-server.log*
do do

1
docker/test/testflows/runner/Dockerfile
View File

@ -21,7 +21,6 @@ RUN apt-get update \
cgroupfs-mount \ cgroupfs-mount \
python3-pip \ python3-pip \
tzdata \ tzdata \
libreadline-dev \
libicu-dev \ libicu-dev \
bsdutils \ bsdutils \
curl \ curl \

79
docs/en/engines/database-engines/materialized-mysql.md
View File

@ -17,6 +17,7 @@ ClickHouse server works as MySQL replica. It reads binlog and performs DDL and D
``` sql ``` sql
CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...] ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
[TABLE OVERRIDE table1 (...), TABLE OVERRIDE table2 (...)]
``` ```
**Engine Parameters** **Engine Parameters**
@ -109,15 +110,19 @@ MySQL DDL queries are converted into the corresponding ClickHouse DDL queries ([
- MySQL `DELETE` query is converted into `INSERT` with `_sign=-1`. - MySQL `DELETE` query is converted into `INSERT` with `_sign=-1`.
- MySQL `UPDATE` query is converted into `INSERT` with `_sign=-1` and `INSERT` with `_sign=1`. - MySQL `UPDATE` query is converted into `INSERT` with `_sign=-1` and `INSERT` with `_sign=1` if the primary key has been changed, or
`INSERT` with `_sign=1` if not.
### Selecting from MaterializedMySQL Tables {#select} ### Selecting from MaterializedMySQL Tables {#select}
`SELECT` query from `MaterializedMySQL` tables has some specifics: `SELECT` query from `MaterializedMySQL` tables has some specifics:
- If `_version` is not specified in the `SELECT` query, [FINAL](../../sql-reference/statements/select/from.md#select-from-final) modifier is used. So only rows with `MAX(_version)` are selected. - If `_version` is not specified in the `SELECT` query, the
[FINAL](../../sql-reference/statements/select/from.md#select-from-final) modifier is used, so only rows with
`MAX(_version)` are returned for each primary key value.
- If `_sign` is not specified in the `SELECT` query, `WHERE _sign=1` is used by default. So the deleted rows are not included into the result set. - If `_sign` is not specified in the `SELECT` query, `WHERE _sign=1` is used by default. So the deleted rows are not
included into the result set.
- The result includes columns comments in case they exist in MySQL database tables. - The result includes columns comments in case they exist in MySQL database tables.
@ -125,15 +130,77 @@ MySQL DDL queries are converted into the corresponding ClickHouse DDL queries ([
MySQL `PRIMARY KEY` and `INDEX` clauses are converted into `ORDER BY` tuples in ClickHouse tables. MySQL `PRIMARY KEY` and `INDEX` clauses are converted into `ORDER BY` tuples in ClickHouse tables.
ClickHouse has only one physical order, which is determined by `ORDER BY` clause. To create a new physical order, use [materialized views](../../sql-reference/statements/create/view.md#materialized). ClickHouse has only one physical order, which is determined by `ORDER BY` clause. To create a new physical order, use
[materialized views](../../sql-reference/statements/create/view.md#materialized).
**Notes** **Notes**
- Rows with `_sign=-1` are not deleted physically from the tables. - Rows with `_sign=-1` are not deleted physically from the tables.
- Cascade `UPDATE/DELETE` queries are not supported by the `MaterializedMySQL` engine. - Cascade `UPDATE/DELETE` queries are not supported by the `MaterializedMySQL` engine, as they are not visible in the
MySQL binlog.
- Replication can be easily broken. - Replication can be easily broken.
- Manual operations on database and tables are forbidden. - Manual operations on database and tables are forbidden.
- `MaterializedMySQL` is influenced by [optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert) setting. The data is merged in the corresponding table in the `MaterializedMySQL` database when a table in the MySQL server changes. - `MaterializedMySQL` is affected by the [optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert)
setting. Data is merged in the corresponding table in the `MaterializedMySQL` database when a table in the MySQL
server changes.
### Table Overrides {#table-overrides}
Table overrides can be used to customize the ClickHouse DDL queries, allowing you to make schema optimizations for your
application. This is especially useful for controlling partitioning, which is important for the overall performance of
MaterializedMySQL.
```sql
CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
[SETTINGS ...]
[TABLE OVERRIDE table_name (
[COLUMNS (
[name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], ...]
[INDEX index_name1 expr1 TYPE type1(...) GRANULARITY value1, ...]
[PROJECTION projection_name_1 (SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]), ...]
)]
[ORDER BY expr]
[PRIMARY KEY expr]
[PARTITION BY expr]
[SAMPLE BY expr]
[TTL expr]
), ...]
```
Example:
```sql
CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
TABLE OVERRIDE table1 (
COLUMNS (
userid UUID,
category LowCardinality(String),
timestamp DateTime CODEC(Delta, Default)
)
PARTITION BY toYear(timestamp)
),
TABLE OVERRIDE table2 (
COLUMNS (
ip_hash UInt32 MATERIALIZED xxHash32(client_ip),
client_ip String TTL created + INTERVAL 72 HOUR
)
SAMPLE BY ip_hash
)
```
The `COLUMNS` list is sparse; it contains only modified or extra (MATERIALIZED or ALIAS) columns. Modified columns with
a different type must be assignable from the original type. There is currently no validation of this or similar issues
when the `CREATE DATABASE` query executes, so extra care needs to be taken.
You may specify overrides for tables that do not exist yet.
!!! note "Warning"
It is easy to break replication with TABLE OVERRIDEs if not used with care. For example:
* If a column is added with a table override, but then later added to the source MySQL table, the converted ALTER TABLE
query in ClickHouse will fail because the column already exists.
* It is currently possible to add overrides that reference nullable columns where not-nullable are required, such as in
`ORDER BY` or `PARTITION BY`.
## Examples of Use {#examples-of-use} ## Examples of Use {#examples-of-use}

93
docs/en/engines/table-engines/special/distributed.md
View File

@ -8,24 +8,43 @@ toc_title: Distributed
Tables with Distributed engine do not store any data of their own, but allow distributed query processing on multiple servers. Tables with Distributed engine do not store any data of their own, but allow distributed query processing on multiple servers.
Reading is automatically parallelized. During a read, the table indexes on remote servers are used, if there are any. Reading is automatically parallelized. During a read, the table indexes on remote servers are used, if there are any.
The Distributed engine accepts parameters: ## Creating a Table {#distributed-creating-a-table}
- the cluster name in the server’s config file ``` sql
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
(
name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1],
name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2],
...
) ENGINE = Distributed(cluster, database, table[, sharding_key[, policy_name]])
[SETTINGS name=value, ...]
```
- the name of a remote database ### From a Table {#distributed-from-a-table}
When the `Distributed` table is pointing to a table on the current server you can adopt that table's schema:
- the name of a remote table ``` sql
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] AS [db2.]name2 ENGINE = Distributed(cluster, database, table[, sharding_key[, policy_name]]) [SETTINGS name=value, ...]
```
- (optionally) sharding key **Distributed Parameters**
- (optionally) policy name, it will be used to store temporary files for async send - `cluster` - the cluster name in the server’s config file
See also: - `database` - the name of a remote database
- `table` - the name of a remote table
- `sharding_key` - (optionally) sharding key
- `policy_name` - (optionally) policy name, it will be used to store temporary files for async send
See also:
- [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) setting - [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) setting
- [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) for the examples - [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) for the examples
Also, it accepts the following settings: **Distributed Settings**
- `fsync_after_insert` - do the `fsync` for the file data after asynchronous insert to Distributed. Guarantees that the OS flushed the whole inserted data to a file **on the initiator node** disk. - `fsync_after_insert` - do the `fsync` for the file data after asynchronous insert to Distributed. Guarantees that the OS flushed the whole inserted data to a file **on the initiator node** disk.
@ -59,24 +78,25 @@ Also, it accepts the following settings:
- [prefer_localhost_replica](../../../operations/settings/settings.md#settings-prefer-localhost-replica) setting - [prefer_localhost_replica](../../../operations/settings/settings.md#settings-prefer-localhost-replica) setting
- `bytes_to_throw_insert` handled before `bytes_to_delay_insert`, so you should not set it to the value less then `bytes_to_delay_insert` - `bytes_to_throw_insert` handled before `bytes_to_delay_insert`, so you should not set it to the value less then `bytes_to_delay_insert`
Example: **Example**
``` sql ``` sql
Distributed(logs, default, hits[, sharding_key[, policy_name]]) CREATE TABLE hits_all AS hits
ENGINE = Distributed(logs, default, hits[, sharding_key[, policy_name]])
SETTINGS SETTINGS
fsync_after_insert=0, fsync_after_insert=0,
fsync_directories=0; fsync_directories=0;
``` ```
Data will be read from all servers in the `logs` cluster, from the default.hits table located on every server in the cluster. Data will be read from all servers in the `logs` cluster, from the `default.hits` table located on every server in the cluster.
Data is not only read but is partially processed on the remote servers (to the extent that this is possible). Data is not only read but is partially processed on the remote servers (to the extent that this is possible).
For example, for a query with GROUP BY, data will be aggregated on remote servers, and the intermediate states of aggregate functions will be sent to the requestor server. Then data will be further aggregated. For example, for a query with `GROUP BY`, data will be aggregated on remote servers, and the intermediate states of aggregate functions will be sent to the requestor server. Then data will be further aggregated.
Instead of the database name, you can use a constant expression that returns a string. For example: currentDatabase(). Instead of the database name, you can use a constant expression that returns a string. For example: `currentDatabase()`.
logs – The cluster name in the server’s config file. ## Clusters {#distributed-clusters}
Clusters are set like this: Clusters are configured in the [server configuration file](../../../operations/configuration-files.md):
``` xml ``` xml
<remote_servers> <remote_servers>
@ -132,12 +152,13 @@ Replicas are duplicating servers (in order to read all the data, you can access
Cluster names must not contain dots. Cluster names must not contain dots.
The parameters `host`, `port`, and optionally `user`, `password`, `secure`, `compression` are specified for each server: The parameters `host`, `port`, and optionally `user`, `password`, `secure`, `compression` are specified for each server:
- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server does not start. If you change the DNS record, restart the server. - `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server does not start. If you change the DNS record, restart the server.
- `port` – The TCP port for messenger activity (`tcp_port` in the config, usually set to 9000). Do not confuse it with http_port. - `port` – The TCP port for messenger activity (`tcp_port` in the config, usually set to 9000). Not to be confused with `http_port`.
- `user` – Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section [Access rights](../../../operations/access-rights.md). - `user` – Name of the user for connecting to a remote server. Default value is the `default` user. This user must have access to connect to the specified server. Access is configured in the `users.xml` file. For more information, see the section [Access rights](../../../operations/access-rights.md).
- `password` – The password for connecting to a remote server (not masked). Default value: empty string. - `password` – The password for connecting to a remote server (not masked). Default value: empty string.
- `secure` - Use ssl for connection, usually you also should define `port` = 9440. Server should listen on `<tcp_port_secure>9440</tcp_port_secure>` and have correct certificates. - `secure` - Whether to use a secure SSL/TLS connection. Usually also requires specifying the port (the default secure port is `9440`). The server should listen on `<tcp_port_secure>9440</tcp_port_secure>` and be configured with correct certificates.
- `compression` - Use data compression. Default value: true. - `compression` - Use data compression. Default value: `true`.
When specifying replicas, one of the available replicas will be selected for each of the shards when reading. You can configure the algorithm for load balancing (the preference for which replica to access) – see the [load_balancing](../../../operations/settings/settings.md#settings-load_balancing) setting. When specifying replicas, one of the available replicas will be selected for each of the shards when reading. You can configure the algorithm for load balancing (the preference for which replica to access) – see the [load_balancing](../../../operations/settings/settings.md#settings-load_balancing) setting.
If the connection with the server is not established, there will be an attempt to connect with a short timeout. If the connection failed, the next replica will be selected, and so on for all the replicas. If the connection attempt failed for all the replicas, the attempt will be repeated the same way, several times. If the connection with the server is not established, there will be an attempt to connect with a short timeout. If the connection failed, the next replica will be selected, and so on for all the replicas. If the connection attempt failed for all the replicas, the attempt will be repeated the same way, several times.
@ -149,40 +170,42 @@ You can specify as many clusters as you wish in the configuration.
To view your clusters, use the `system.clusters` table. To view your clusters, use the `system.clusters` table.
The Distributed engine allows working with a cluster like a local server. However, the cluster is inextensible: you must write its configuration in the server config file (even better, for all the cluster’s servers). The `Distributed` engine allows working with a cluster like a local server. However, the cluster's configuration cannot be specified dynamically, it has to be configured in the server config file. Usually, all servers in a cluster will have the same cluster config (though this is not required). Clusters from the config file are updated on the fly, without restarting the server.
The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you do not need to create a Distributed table – use the `remote` table function instead. See the section [Table functions](../../../sql-reference/table-functions/index.md). If you need to send a query to an unknown set of shards and replicas each time, you do not need to create a `Distributed` table – use the `remote` table function instead. See the section [Table functions](../../../sql-reference/table-functions/index.md).
## Writing data {#distributed-writing-data}
There are two methods for writing data to a cluster: There are two methods for writing data to a cluster:
First, you can define which servers to write which data to and perform the write directly on each shard. In other words, perform INSERT in the tables that the distributed table “looks at”. This is the most flexible solution as you can use any sharding scheme, which could be non-trivial due to the requirements of the subject area. This is also the most optimal solution since data can be written to different shards completely independently. First, you can define which servers to write which data to and perform the write directly on each shard. In other words, perform direct `INSERT` statements on the remote tables in the cluster that the `Distributed` table is pointing to. This is the most flexible solution as you can use any sharding scheme, even one that is non-trivial due to the requirements of the subject area. This is also the most optimal solution since data can be written to different shards completely independently.
Second, you can perform INSERT in a Distributed table. In this case, the table will distribute the inserted data across the servers itself. In order to write to a Distributed table, it must have a sharding key set (the last parameter). In addition, if there is only one shard, the write operation works without specifying the sharding key, since it does not mean anything in this case. Second, you can perform `INSERT` statements on a `Distributed` table. In this case, the table will distribute the inserted data across the servers itself. In order to write to a `Distributed` table, it must have the `sharding_key` parameter configured (except if there is only one shard).
Each shard can have a weight defined in the config file. By default, the weight is equal to one. Data is distributed across shards in the amount proportional to the shard weight. For example, if there are two shards and the first has a weight of 9 while the second has a weight of 10, the first will be sent 9 / 19 parts of the rows, and the second will be sent 10 / 19. Each shard can have a `<weight>` defined in the config file. By default, the weight is `1`. Data is distributed across shards in the amount proportional to the shard weight. All shard weights are summed up, then each shard's weight is divided by the total to determine each shard's proportion. For example, if there are two shards and the first has a weight of 1 while the second has a weight of 2, the first will be sent one third (1 / 3) of inserted rows and the second will be sent two thirds (2 / 3).
Each shard can have the `internal_replication` parameter defined in the config file. Each shard can have the `internal_replication` parameter defined in the config file. If this parameter is set to `true`, the write operation selects the first healthy replica and writes data to it. Use this if the tables underlying the `Distributed` table are replicated tables (e.g. any of the `Replicated*MergeTree` table engines). One of the table replicas will receive the write and it will be replicated to the other replicas automatically.
If this parameter is set to `true`, the write operation selects the first healthy replica and writes data to it. Use this alternative if the Distributed table “looks at” replicated tables. In other words, if the table where data will be written is going to replicate them itself. If `internal_replication` is set to `false` (the default), data is written to all replicas. In this case, the `Distributed` table replicates data itself. This is worse than using replicated tables because the consistency of replicas is not checked and, over time, they will contain slightly different data.
If it is set to `false` (the default), data is written to all replicas. In essence, this means that the Distributed table replicates data itself. This is worse than using replicated tables, because the consistency of replicas is not checked, and over time they will contain slightly different data.
To select the shard that a row of data is sent to, the sharding expression is analyzed, and its remainder is taken from dividing it by the total weight of the shards. The row is sent to the shard that corresponds to the half-interval of the remainders from `prev_weights` to `prev_weights + weight`, where `prev_weights` is the total weight of the shards with the smallest number, and `weight` is the weight of this shard. For example, if there are two shards, and the first has a weight of 9 while the second has a weight of 10, the row will be sent to the first shard for the remainders from the range \[0, 9), and to the second for the remainders from the range \[9, 19). To select the shard that a row of data is sent to, the sharding expression is analyzed, and its remainder is taken from dividing it by the total weight of the shards. The row is sent to the shard that corresponds to the half-interval of the remainders from `prev_weights` to `prev_weights + weight`, where `prev_weights` is the total weight of the shards with the smallest number, and `weight` is the weight of this shard. For example, if there are two shards, and the first has a weight of 9 while the second has a weight of 10, the row will be sent to the first shard for the remainders from the range \[0, 9), and to the second for the remainders from the range \[9, 19).
The sharding expression can be any expression from constants and table columns that returns an integer. For example, you can use the expression `rand()` for random distribution of data, or `UserID` for distribution by the remainder from dividing the user’s ID (then the data of a single user will reside on a single shard, which simplifies running IN and JOIN by users). If one of the columns is not distributed evenly enough, you can wrap it in a hash function: intHash64(UserID). The sharding expression can be any expression from constants and table columns that returns an integer. For example, you can use the expression `rand()` for random distribution of data, or `UserID` for distribution by the remainder from dividing the user’s ID (then the data of a single user will reside on a single shard, which simplifies running `IN` and `JOIN` by users). If one of the columns is not distributed evenly enough, you can wrap it in a hash function e.g. `intHash64(UserID)`.
A simple remainder from the division is a limited solution for sharding and isn’t always appropriate. It works for medium and large volumes of data (dozens of servers), but not for very large volumes of data (hundreds of servers or more). In the latter case, use the sharding scheme required by the subject area, rather than using entries in Distributed tables. A simple remainder from the division is a limited solution for sharding and isn’t always appropriate. It works for medium and large volumes of data (dozens of servers), but not for very large volumes of data (hundreds of servers or more). In the latter case, use the sharding scheme required by the subject area rather than using entries in `Distributed` tables.
SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you do not have to transfer old data into it. Instead, you can write new data to it by using a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently.
You should be concerned about the sharding scheme in the following cases: You should be concerned about the sharding scheme in the following cases:
- Queries are used that require joining data (IN or JOIN) by a specific key. If data is sharded by this key, you can use local IN or JOIN instead of GLOBAL IN or GLOBAL JOIN, which is much more efficient. - Queries are used that require joining data (`IN` or `JOIN`) by a specific key. If data is sharded by this key, you can use local `IN` or `JOIN` instead of `GLOBAL IN` or `GLOBAL JOIN`, which is much more efficient.
- A large number of servers is used (hundreds or more) with a large number of small queries (queries of individual clients - websites, advertisers, or partners). In order for the small queries to not affect the entire cluster, it makes sense to locate data for a single client on a single shard. Alternatively, as we’ve done in Yandex.Metrica, you can set up bi-level sharding: divide the entire cluster into “layers”, where a layer may consist of multiple shards. Data for a single client is located on a single layer, but shards can be added to a layer as necessary, and data is randomly distributed within them. Distributed tables are created for each layer, and a single shared distributed table is created for global queries. - A large number of servers is used (hundreds or more) with a large number of small queries, for example, queries for data of individual clients (e.g. websites, advertisers, or partners). In order for the small queries to not affect the entire cluster, it makes sense to locate data for a single client on a single shard. Alternatively, as we’ve done in Yandex.Metrica, you can set up bi-level sharding: divide the entire cluster into “layers”, where a layer may consist of multiple shards. Data for a single client is located on a single layer, but shards can be added to a layer as necessary, and data is randomly distributed within them. `Distributed` tables are created for each layer, and a single shared distributed table is created for global queries.
Data is written asynchronously. When inserted in the table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. The periodicity for sending data is managed by the [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) and [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) settings. The `Distributed` engine sends each file with inserted data separately, but you can enable batch sending of files with the [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) setting. This setting improves cluster performance by better utilizing local server and network resources. You should check whether data is sent successfully by checking the list of files (data waiting to be sent) in the table directory: `/var/lib/clickhouse/data/database/table/`. The number of threads performing background tasks can be set by [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) setting. Data is written asynchronously. When inserted in the table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. The periodicity for sending data is managed by the [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) and [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) settings. The `Distributed` engine sends each file with inserted data separately, but you can enable batch sending of files with the [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) setting. This setting improves cluster performance by better utilizing local server and network resources. You should check whether data is sent successfully by checking the list of files (data waiting to be sent) in the table directory: `/var/lib/clickhouse/data/database/table/`. The number of threads performing background tasks can be set by [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) setting.
If the server ceased to exist or had a rough restart (for example, after a device failure) after an INSERT to a Distributed table, the inserted data might be lost. If a damaged data part is detected in the table directory, it is transferred to the `broken` subdirectory and no longer used. If the server ceased to exist or had a rough restart (for example, due to a hardware failure) after an `INSERT` to a `Distributed` table, the inserted data might be lost. If a damaged data part is detected in the table directory, it is transferred to the `broken` subdirectory and no longer used.
## Reading data {#distributed-reading-data}
When querying a `Distributed` table, `SELECT` queries are sent to all shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you do not have to transfer old data into it. Instead, you can write new data to it by using a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently.
When the `max_parallel_replicas` option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section [max_parallel_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). When the `max_parallel_replicas` option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section [max_parallel_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas).

99
docs/en/interfaces/grpc.md Normal file
View File

@ -0,0 +1,99 @@
---
toc_priority: 19
toc_title: gRPC Interface
---
# gRPC Interface {#grpc-interface}
## Introduction {#grpc-interface-introduction}
ClickHouse supports [gRPC](https://grpc.io/) interface. It is an open source remote procedure call system that uses HTTP/2 and [Protocol Buffers](https://en.wikipedia.org/wiki/Protocol_Buffers). The implementation of gRPC in ClickHouse supports:
- SSL;
- authentication;
- sessions;
- compression;
- parallel queries through the same channel;
- cancellation of queries;
- getting progress and logs;
- external tables.
The specification of the interface is described in [clickhouse_grpc.proto](https://github.com/ClickHouse/ClickHouse/blob/master/src/Server/grpc_protos/clickhouse_grpc.proto).
## gRPC Configuration {#grpc-interface-configuration}
To use the gRPC interface set `grpc_port` in the main [server configuration](../operations/configuration-files.md). Other configuration options see in the following example:
```xml
<grpc_port>9100</grpc_port>
<grpc>
<enable_ssl>false</enable_ssl>
<!-- The following two files are used only if SSL is enabled -->
<ssl_cert_file>/path/to/ssl_cert_file</ssl_cert_file>
<ssl_key_file>/path/to/ssl_key_file</ssl_key_file>
<!-- Whether server requests client for a certificate -->
<ssl_require_client_auth>false</ssl_require_client_auth>
<!-- The following file is used only if ssl_require_client_auth=true -->
<ssl_ca_cert_file>/path/to/ssl_ca_cert_file</ssl_ca_cert_file>
<!-- Default compression algorithm (applied if client doesn't specify another algorithm, see result_compression in QueryInfo).
Supported algorithms: none, deflate, gzip, stream_gzip -->
<compression>deflate</compression>
<!-- Default compression level (applied if client doesn't specify another level, see result_compression in QueryInfo).
Supported levels: none, low, medium, high -->
<compression_level>medium</compression_level>
<!-- Send/receive message size limits in bytes. -1 means unlimited -->
<max_send_message_size>-1</max_send_message_size>
<max_receive_message_size>-1</max_receive_message_size>
<!-- Enable if you want to get detailed logs -->
<verbose_logs>false</verbose_logs>
</grpc>
```
## Built-in Client {#grpc-client}
You can write a client in any of the programming languages supported by gRPC using the provided [specification](https://github.com/ClickHouse/ClickHouse/blob/master/src/Server/grpc_protos/clickhouse_grpc.proto).
Or you can use a built-in Python client. It is placed in [utils/grpc-client/clickhouse-grpc-client.py](https://github.com/ClickHouse/ClickHouse/blob/master/utils/grpc-client/clickhouse-grpc-client.py) in the repository. The built-in client requires [grpcio and grpcio-tools](https://grpc.io/docs/languages/python/quickstart) Python modules.
The client supports the following arguments:
- `--help` – Shows a help message and exits.
- `--host HOST, -h HOST` – A server name. Default value: `localhost`. You can use IPv4 or IPv6 addresses also.
- `--port PORT` – A port to connect to. This port should be enabled in the ClickHouse server configuration (see `grpc_port`). Default value: `9100`.
- `--user USER_NAME, -u USER_NAME` – A user name. Default value: `default`.
- `--password PASSWORD` – A password. Default value: empty string.
- `--query QUERY, -q QUERY` – A query to process when using non-interactive mode.
- `--database DATABASE, -d DATABASE` – A default database. If not specified, the current database set in the server settings is used (`default` by default).
- `--format OUTPUT_FORMAT, -f OUTPUT_FORMAT` – A result output [format](formats.md). Default value for interactive mode: `PrettyCompact`.
- `--debug` – Enables showing debug information.
To run the client in an interactive mode call it without `--query` argument.
In a batch mode query data can be passed via `stdin`.
**Client Usage Example**
In the following example a table is created and loaded with data from a CSV file. Then the content of the table is queried.
``` bash
./clickhouse-grpc-client.py -q "CREATE TABLE grpc_example_table (id UInt32, text String) ENGINE = MergeTree() ORDER BY id;"
echo "0,Input data for" > a.txt ; echo "1,gRPC protocol example" >> a.txt
cat a.txt | ./clickhouse-grpc-client.py -q "INSERT INTO grpc_example_table FORMAT CSV"
./clickhouse-grpc-client.py --format PrettyCompact -q "SELECT * FROM grpc_example_table;"
```
Result:
``` text
┌─id─┬─text──────────────────┐
│ 0 │ Input data for │
│ 1 │ gRPC protocol example │
└────┴───────────────────────┘
```

4
docs/en/interfaces/index.md
View File

@ -6,10 +6,11 @@ toc_title: Introduction
# Interfaces {#interfaces} # Interfaces {#interfaces}
ClickHouse provides two network interfaces (both can be optionally wrapped in TLS for additional security): ClickHouse provides three network interfaces (they can be optionally wrapped in TLS for additional security):
- [HTTP](http.md), which is documented and easy to use directly. - [HTTP](http.md), which is documented and easy to use directly.
- [Native TCP](../interfaces/tcp.md), which has less overhead. - [Native TCP](../interfaces/tcp.md), which has less overhead.
- [gRPC](grpc.md).
In most cases it is recommended to use appropriate tool or library instead of interacting with those directly. Officially supported by Yandex are the following: In most cases it is recommended to use appropriate tool or library instead of interacting with those directly. Officially supported by Yandex are the following:
@ -24,4 +25,3 @@ There are also a wide range of third-party libraries for working with ClickHouse
- [Integrations](../interfaces/third-party/integrations.md) - [Integrations](../interfaces/third-party/integrations.md)
- [Visual interfaces](../interfaces/third-party/gui.md) - [Visual interfaces](../interfaces/third-party/gui.md)
[Original article](https://clickhouse.com/docs/en/interfaces/) <!--hide-->

29
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -505,7 +505,7 @@ Keys:
- `level` – Logging level. Acceptable values: `trace`, `debug`, `information`, `warning`, `error`. - `level` – Logging level. Acceptable values: `trace`, `debug`, `information`, `warning`, `error`.
- `log` – The log file. Contains all the entries according to `level`. - `log` – The log file. Contains all the entries according to `level`.
- `errorlog` – Error log file. - `errorlog` – Error log file.
- `size` – Size of the file. Applies to `log`and`errorlog`. Once the file reaches `size`, ClickHouse archives and renames it, and creates a new log file in its place. - `size` – Size of the file. Applies to `log` and `errorlog`. Once the file reaches `size`, ClickHouse archives and renames it, and creates a new log file in its place.
- `count` – The number of archived log files that ClickHouse stores. - `count` – The number of archived log files that ClickHouse stores.
**Example** **Example**
@ -750,9 +750,13 @@ The value 0 means that you can delete all tables without any restrictions.
## max_thread_pool_size {#max-thread-pool-size} ## max_thread_pool_size {#max-thread-pool-size}
The maximum number of threads in the Global Thread pool. ClickHouse uses threads from the Global Thread pool to process queries. If there is no idle thread to process a query, then a new thread is created in the pool. `max_thread_pool_size` limits the maximum number of threads in the pool.
Default value: 10000. Possible values:
- Positive integer.
Default value: `10000`.
**Example** **Example**
@ -762,9 +766,13 @@ Default value: 10000.
## max_thread_pool_free_size {#max-thread-pool-free-size} ## max_thread_pool_free_size {#max-thread-pool-free-size}
The number of threads that are always held in the Global Thread pool. If the number of **idle** threads in the Global Thread pool is greater than `max_thread_pool_free_size`, then ClickHouse releases resources occupied by some threads and the pool size is decreased. Threads can be created again if necessary.
Default value: 1000. Possible values:
- Positive integer.
Default value: `1000`.
**Example** **Example**
@ -774,9 +782,13 @@ Default value: 1000.
## thread_pool_queue_size {#thread-pool-queue-size} ## thread_pool_queue_size {#thread-pool-queue-size}
The limit to the number of jobs that can be scheduled on the Global Thread pool. Increasing queue size leads to larger memory usage. It is recommended to keep this value equal to the `max_thread_pool_size`. The maximum number of jobs that can be scheduled on the Global Thread pool. Increasing queue size leads to larger memory usage. It is recommended to keep this value equal to [max_thread_pool_size](#max-thread-pool-size).
Default value: 10000. Possible values:
- Positive integer.
Default value: `10000`.
**Example** **Example**
@ -1443,7 +1455,7 @@ You can also define sections `memory` — means storing information only in memo
To add an LDAP server as a remote user directory of users that are not defined locally, define a single `ldap` section with a following parameters: To add an LDAP server as a remote user directory of users that are not defined locally, define a single `ldap` section with a following parameters:
- `server` — one of LDAP server names defined in `ldap_servers` config section. This parameter is mandatory and cannot be empty. - `server` — one of LDAP server names defined in `ldap_servers` config section. This parameter is mandatory and cannot be empty.
- `roles` — section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. If no roles are specified, user will not be able to perform any actions after authentication. If any of the listed roles is not defined locally at the time of authentication, the authenthication attept will fail as if the provided password was incorrect. - `roles` — section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. If no roles are specified, user will not be able to perform any actions after authentication. If any of the listed roles is not defined locally at the time of authentication, the authentication attempt will fail as if the provided password was incorrect.
**Example** **Example**
@ -1507,3 +1519,4 @@ Possible values:
- Positive integer. - Positive integer.
Default value: `10000`. Default value: `10000`.

21
docs/en/operations/settings/merge-tree-settings.md
View File

@ -356,3 +356,24 @@ Possible values:
- 1 — Parts are detached. - 1 — Parts are detached.
Default value: `0`. Default value: `0`.
## merge_tree_clear_old_temporary_directories_interval_seconds {#setting-merge-tree-clear-old-temporary-directories-interval-seconds}
Sets the interval in seconds for ClickHouse to execute the cleanup of old temporary directories.
Possible values:
- Any positive integer.
Default value: `60` seconds.
## merge_tree_clear_old_parts_interval_seconds {#setting-merge-tree-clear-old-parts-interval-seconds}
Sets the interval in seconds for ClickHouse to execute the cleanup of old parts, WALs, and mutations.
Possible values:
- Any positive integer.
Default value: `1` second.

20
docs/en/operations/settings/settings.md
View File

@ -885,26 +885,6 @@ Possible values:
Default value: 2013265920. Default value: 2013265920.
## merge_tree_clear_old_temporary_directories_interval_seconds {#setting-merge-tree-clear-old-temporary-directories-interval-seconds}
Sets the interval in seconds for ClickHouse to execute the cleanup of old temporary directories.
Possible values:
- Any positive integer.
Default value: `60` seconds.
## merge_tree_clear_old_parts_interval_seconds {#setting-merge-tree-clear-old-parts-interval-seconds}
Sets the interval in seconds for ClickHouse to execute the cleanup of old parts, WALs, and mutations.
Possible values:
- Any positive integer.
Default value: `1` second.
## min_bytes_to_use_direct_io {#settings-min-bytes-to-use-direct-io} ## min_bytes_to_use_direct_io {#settings-min-bytes-to-use-direct-io}
The minimum data volume required for using direct I/O access to the storage disk. The minimum data volume required for using direct I/O access to the storage disk.

2
docs/en/operations/tips.md
View File

@ -34,7 +34,7 @@ Use `perf top` to watch the time spent in the kernel for memory management.
Permanent huge pages also do not need to be allocated. Permanent huge pages also do not need to be allocated.
!!! warning "Attention" !!! warning "Attention"
If your system has less than 16 GB of RAM you may experience various memory exceptions because default settings does not match this amount of RAM. Recommended amount of RAM is 32 GB or more. You can use ClickHouse in system with small amount of RAM, even with 2 GB of RAM, but it requires an additional tuning and able to process small ingestion rate. If your system has less than 16 GB of RAM, you may experience various memory exceptions because default settings do not match this amount of memory. The recommended amount of RAM is 32 GB or more. You can use ClickHouse in a system with a small amount of RAM, even with 2 GB of RAM, but it requires additional tuning and can ingest at a low rate.
## Storage Subsystem {#storage-subsystem} ## Storage Subsystem {#storage-subsystem}

2
docs/en/operations/utilities/odbc-bridge.md
View File

@ -26,7 +26,7 @@ Query is send in post body. Response is returned in RowBinary format.
```bash ```bash
$ clickhouse-odbc-bridge --http-port 9018 --daemon $ clickhouse-odbc-bridge --http-port 9018 --daemon
$ curl -d "query=SELECT PageID, ImpID, AdType FROM Keys ORDER BY PageID, ImpID" --data-urlencode "connection_string=DSN=ClickHouse;DATABASE=stat" --data-urlencode "columns=columns format version: 1 $ curl -d "query=SELECT PageID, ImpID, AdType FROM Keys ORDER BY PageID, ImpID" --data-urlencode "connection_string=DSN=ClickHouse;DATABASE=stat" --data-urlencode "sample_block=columns format version: 1
3 columns: 3 columns:
\`PageID\` String \`PageID\` String
\`ImpID\` String \`ImpID\` String

64
docs/en/sql-reference/aggregate-functions/reference/sparkbar.md Normal file
View File

@ -0,0 +1,64 @@
---
toc_priority: 311
toc_title: sparkbar
---
# sparkbar {#sparkbar}
The function plots a frequency histogram for values `x` and the repetition rate `y` of these values over the interval `[min_x, max_x]`.
If no interval is specified, then the minimum `x` is used as the interval start, and the maximum `x` — as the interval end.
**Syntax**
``` sql
sparkbar(width[, min_x, max_x])(x, y)
```
**Parameters**
- `width` — The number of segments. Type: [Integer](../../../sql-reference/data-types/int-uint.md).
- `min_x` — The interval start. Optional parameter.
- `max_x` — The interval end. Optional parameter.
**Arguments**
- `x` — The field with values.
- `y` — The field with the frequency of values.
**Returned value**
- The frequency histogram.
**Example**
Query:
``` sql
CREATE TABLE spark_bar_data (`cnt` UInt64,`event_date` Date) ENGINE = MergeTree ORDER BY event_date SETTINGS index_granularity = 8192;
INSERT INTO spark_bar_data VALUES(1,'2020-01-01'),(4,'2020-01-02'),(5,'2020-01-03'),(2,'2020-01-04'),(3,'2020-01-05'),(7,'2020-01-06'),(6,'2020-01-07'),(8,'2020-01-08'),(2,'2020-01-11');
SELECT sparkbar(9)(event_date,cnt) FROM spark_bar_data;
SELECT sparkbar(9,toDate('2020-01-01'),toDate('2020-01-10'))(event_date,cnt) FROM spark_bar_data;
```
Result:
``` text
┌─sparkbar(9)(event_date, cnt)─┐
│ │
│ ▁▅▄▃██▅ ▁ │
│ │
└──────────────────────────────┘
┌─sparkbar(9, toDate('2020-01-01'), toDate('2020-01-10'))(event_date, cnt)─┐
│ │
│▁▄▄▂▅▇█▁ │
│ │
└──────────────────────────────────────────────────────────────────────────┘
```

34
docs/en/sql-reference/functions/window-view-functions.md → docs/en/sql-reference/functions/time-window-functions.md
View File

@ -1,15 +1,15 @@
--- ---
toc_priority: 68 toc_priority: 68
toc_title: Window View toc_title: Time Window
--- ---
# Window View Functions {#window-view-functions} # Time Window Functions {#time-window-functions}
Window functions indicate the lower and upper window bound of records in WindowView. The functions for working with WindowView are listed below. Time window functions return the inclusive lower and exclusive upper bound of the corresponding window. The functions for working with WindowView are listed below:
## tumble {#window-view-functions-tumble} ## tumble {#time-window-functions-tumble}
A tumbling time window assigns records to non-overlapping, continuous windows with a fixed duration (interval). A tumbling time window assigns records to non-overlapping, continuous windows with a fixed duration (`interval`).
``` sql ``` sql
tumble(time_attr, interval [, timezone]) tumble(time_attr, interval [, timezone])
@ -22,7 +22,7 @@ tumble(time_attr, interval [, timezone])
**Returned values** **Returned values**
- The lower and upper bound of the tumble window. - The inclusive lower and exclusive upper bound of the corresponding tumbling window.
Type: `Tuple(DateTime, DateTime)` Type: `Tuple(DateTime, DateTime)`
@ -42,7 +42,7 @@ Result:
└───────────────────────────────────────────────┘ └───────────────────────────────────────────────┘
``` ```
## hop {#window-view-functions-hop} ## hop {#time-window-functions-hop}
A hopping time window has a fixed duration (`window_interval`) and hops by a specified hop interval (`hop_interval`). If the `hop_interval` is smaller than the `window_interval`, hopping windows are overlapping. Thus, records can be assigned to multiple windows. A hopping time window has a fixed duration (`window_interval`) and hops by a specified hop interval (`hop_interval`). If the `hop_interval` is smaller than the `window_interval`, hopping windows are overlapping. Thus, records can be assigned to multiple windows.
@ -59,9 +59,7 @@ hop(time_attr, hop_interval, window_interval [, timezone])
**Returned values** **Returned values**
- The lower and upper bound of the hop window. Since hop windows are - The inclusive lower and exclusive upper bound of the corresponding hopping window. Since one record can be assigned to multiple hop windows, the function only returns the bound of the **first** window when hop function is used **without** `WINDOW VIEW`.
overlapped, the function only returns the bound of the **first** window when
hop function is used **without** `WINDOW VIEW`.
Type: `Tuple(DateTime, DateTime)` Type: `Tuple(DateTime, DateTime)`
@ -81,33 +79,33 @@ Result:
└───────────────────────────────────────────────────────────┘ └───────────────────────────────────────────────────────────┘
``` ```
## tumbleStart {#window-view-functions-tumblestart} ## tumbleStart {#time-window-functions-tumblestart}
Indicate the lower bound of a tumble function. Returns the inclusive lower bound of the corresponding tumbling window.
``` sql ``` sql
tumbleStart(time_attr, interval [, timezone]); tumbleStart(time_attr, interval [, timezone]);
``` ```
## tumbleEnd {#window-view-functions-tumbleend} ## tumbleEnd {#time-window-functions-tumbleend}
Indicate the upper bound of a tumble function. Returns the exclusive upper bound of the corresponding tumbling window.
``` sql ``` sql
tumbleEnd(time_attr, interval [, timezone]); tumbleEnd(time_attr, interval [, timezone]);
``` ```
## hopStart {#window-view-functions-hopstart} ## hopStart {#time-window-functions-hopstart}
Indicate the lower bound of a hop function. Returns the inclusive lower bound of the corresponding hopping window.
``` sql ``` sql
hopStart(time_attr, hop_interval, window_interval [, timezone]); hopStart(time_attr, hop_interval, window_interval [, timezone]);
``` ```
## hopEnd {#window-view-functions-hopend} ## hopEnd {#time-window-functions-hopend}
Indicate the upper bound of a hop function. Returns the exclusive upper bound of the corresponding hopping window.
``` sql ``` sql
hopEnd(time_attr, hop_interval, window_interval [, timezone]); hopEnd(time_attr, hop_interval, window_interval [, timezone]);

25
docs/en/sql-reference/statements/create/view.md
View File

@ -251,22 +251,22 @@ Most common uses of live view tables include:
Enable usage of window views and `WATCH` query using [allow_experimental_window_view](../../../operations/settings/settings.md#allow-experimental-window-view) setting. Input the command `set allow_experimental_window_view = 1`. Enable usage of window views and `WATCH` query using [allow_experimental_window_view](../../../operations/settings/settings.md#allow-experimental-window-view) setting. Input the command `set allow_experimental_window_view = 1`.
``` sql ``` sql
CREATE WINDOW VIEW [IF NOT EXISTS] [db.]table_name [TO [db.]table_name] [ENGINE = engine] [WATERMARK = strategy] [ALLOWED_LATENESS = interval_function] AS SELECT ... GROUP BY window_view_function CREATE WINDOW VIEW [IF NOT EXISTS] [db.]table_name [TO [db.]table_name] [ENGINE = engine] [WATERMARK = strategy] [ALLOWED_LATENESS = interval_function] AS SELECT ... GROUP BY time_window_function
``` ```
Window view can aggregate data by time window and output the results when the window is ready to fire. It stores the partial aggregation results in an inner(or specified) table and can push the processing result to a specified table or push notifications using the WATCH query. Window view can aggregate data by time window and output the results when the window is ready to fire. It stores the partial aggregation results in an inner(or specified) table to reduce latency and can push the processing result to a specified table or push notifications using the WATCH query.
Creating a window view is similar to creating `MATERIALIZED VIEW`. Window view needs an inner storage engine to store intermediate data. The inner storage will use `AggregatingMergeTree` as the default engine. Creating a window view is similar to creating `MATERIALIZED VIEW`. Window view needs an inner storage engine to store intermediate data. The inner storage will use `AggregatingMergeTree` as the default engine.
### Window View Functions {#window-view-windowviewfunctions} ### Time Window Functions {#window-view-timewindowfunctions}
[Window view functions](../../functions/window-view-functions.md) are used to indicate the lower and upper window bound of records. The window view needs to be used with a window view function. [Time window functions](../../functions/time-window-functions.md) are used to get the lower and upper window bound of records. The window view needs to be used with a time window function.
### TIME ATTRIBUTES {#window-view-timeattributes} ### TIME ATTRIBUTES {#window-view-timeattributes}
Window view supports **processing time** and **event time** process. Window view supports **processing time** and **event time** process.
**Processing time** allows window view to produce results based on the local machine's time and is used by default. It is the most straightforward notion of time but does not provide determinism. The processing time attribute can be defined by setting the `time_attr` of the window view function to a table column or using the function `now()`. The following query creates a window view with processing time. **Processing time** allows window view to produce results based on the local machine's time and is used by default. It is the most straightforward notion of time but does not provide determinism. The processing time attribute can be defined by setting the `time_attr` of the time window function to a table column or using the function `now()`. The following query creates a window view with processing time.
``` sql ``` sql
CREATE WINDOW VIEW wv AS SELECT count(number), tumbleStart(w_id) as w_start from date GROUP BY tumble(now(), INTERVAL '5' SECOND) as w_id CREATE WINDOW VIEW wv AS SELECT count(number), tumbleStart(w_id) as w_start from date GROUP BY tumble(now(), INTERVAL '5' SECOND) as w_id
@ -274,13 +274,13 @@ CREATE WINDOW VIEW wv AS SELECT count(number), tumbleStart(w_id) as w_start from
**Event time** is the time that each individual event occurred on its producing device. This time is typically embedded within the records when it is generated. Event time processing allows for consistent results even in case of out-of-order events or late events. Window view supports event time processing by using `WATERMARK` syntax. **Event time** is the time that each individual event occurred on its producing device. This time is typically embedded within the records when it is generated. Event time processing allows for consistent results even in case of out-of-order events or late events. Window view supports event time processing by using `WATERMARK` syntax.
Window view provides three watermark strategies. Window view provides three watermark strategies:
* `STRICTLY_ASCENDING`: Emits a watermark of the maximum observed timestamp so far. Rows that have a timestamp smaller to the max timestamp are not late. * `STRICTLY_ASCENDING`: Emits a watermark of the maximum observed timestamp so far. Rows that have a timestamp smaller to the max timestamp are not late.
* `ASCENDING`: Emits a watermark of the maximum observed timestamp so far minus 1. Rows that have a timestamp equal and smaller to the max timestamp are not late. * `ASCENDING`: Emits a watermark of the maximum observed timestamp so far minus 1. Rows that have a timestamp equal and smaller to the max timestamp are not late.
* `BOUNDED`: WATERMARK=INTERVAL. Emits watermarks, which are the maximum observed timestamp minus the specified delay. * `BOUNDED`: WATERMARK=INTERVAL. Emits watermarks, which are the maximum observed timestamp minus the specified delay.
The following queries are examples of creating a window view with `WATERMARK`. The following queries are examples of creating a window view with `WATERMARK`:
``` sql ``` sql
CREATE WINDOW VIEW wv WATERMARK=STRICTLY_ASCENDING AS SELECT count(number) FROM date GROUP BY tumble(timestamp, INTERVAL '5' SECOND); CREATE WINDOW VIEW wv WATERMARK=STRICTLY_ASCENDING AS SELECT count(number) FROM date GROUP BY tumble(timestamp, INTERVAL '5' SECOND);
@ -296,15 +296,18 @@ CREATE WINDOW VIEW test.wv TO test.dst WATERMARK=ASCENDING ALLOWED_LATENESS=INTE
Note that elements emitted by a late firing should be treated as updated results of a previous computation. Instead of firing at the end of windows, the window view will fire immediately when the late event arrives. Thus, it will result in multiple outputs for the same window. Users need to take these duplicated results into account or deduplicate them. Note that elements emitted by a late firing should be treated as updated results of a previous computation. Instead of firing at the end of windows, the window view will fire immediately when the late event arrives. Thus, it will result in multiple outputs for the same window. Users need to take these duplicated results into account or deduplicate them.
### Monitoring New Windows{#window-view-monitoring} ### Monitoring New Windows {#window-view-monitoring}
Window view supports the `WATCH` query to constantly append the processing results to the console or use `TO` syntax to output the results to a table. Window view supports the [WATCH](../../../sql-reference/statements/watch.md) query to monitoring changes, or use `TO` syntax to output the results to a table.
``` sql ``` sql
WATCH [db.]name [LIMIT n] WATCH [db.]window_view
[EVENTS]
[LIMIT n]
[FORMAT format]
``` ```
`WATCH` query acts similar as in `LIVE VIEW`. A `LIMIT` can be specified to set the number of updates to receive before terminating the query. `WATCH` query acts similar as in `LIVE VIEW`. A `LIMIT` can be specified to set the number of updates to receive before terminating the query. The `EVENTS` clause can be used to obtain a short form of the `WATCH` query where instead of the query result you will just get the latest query watermark.
### Settings {#window-view-settings} ### Settings {#window-view-settings}

99
docs/ru/interfaces/grpc.md Normal file
View File

@ -0,0 +1,99 @@
---
toc_priority: 18
toc_title: gRPC интерфейс
---
# Интерфейс gRPC {#grpc-interface}
## Введение {#grpc-interface-introduction}
ClickHouse поддерживает интерфейс [gRPC](https://grpc.io/). Это система удаленного вызова процедур с открытым исходным кодом, которая использует HTTP/2 и [Protocol Buffers](https://ru.wikipedia.org/wiki/Protocol_Buffers). В реализации gRPC в ClickHouse поддерживаются:
- SSL;
- аутентификация;
- сессии;
- сжатие;
- параллельные запросы, выполняемые через один канал;
- отмена запросов;
- получение прогресса операций и логов;
- внешние таблицы.
Спецификация интерфейса содержится в [clickhouse_grpc.proto](https://github.com/ClickHouse/ClickHouse/blob/master/src/Server/grpc_protos/clickhouse_grpc.proto).
## Конфигурация gRPC {#grpc-interface-configuration}
Чтобы сделать доступным интерфейс gRPC, нужно задать порт с помощью настройки `grpc_port` в [конфигурации сервера](../operations/configuration-files.md). Другие настройки приведены в примере:
```xml
<grpc_port>9100</grpc_port>
<grpc>
<enable_ssl>false</enable_ssl>
<!-- Пути к файлам сертификатов и ключей. Используются при включенном SSL -->
<ssl_cert_file>/path/to/ssl_cert_file</ssl_cert_file>
<ssl_key_file>/path/to/ssl_key_file</ssl_key_file>
<!-- Запрашивает ли сервер сертификат клиента -->
<ssl_require_client_auth>false</ssl_require_client_auth>
<!-- Используется, если необходимо запрашивать сертификат -->
<ssl_ca_cert_file>/path/to/ssl_ca_cert_file</ssl_ca_cert_file>
<!-- Алгоритм сжатия по умолчанию (применяется, если клиент не указывает алгоритм, см. result_compression в QueryInfo).
Поддерживаются алгоритмы: none, deflate, gzip, stream_gzip -->
<compression>deflate</compression>
<!-- Уровень сжатия по умолчанию (применяется, если клиент не указывает уровень сжатия, см. result_compression в QueryInfo).
Поддерживаемые уровни: none, low, medium, high -->
<compression_level>medium</compression_level>
<!-- Ограничение в байтах на размер отправляемых и принимаемых сообщений. -1 означает отсутствие ограничения -->
<max_send_message_size>-1</max_send_message_size>
<max_receive_message_size>-1</max_receive_message_size>
<!-- Выводить ли детализированные логи -->
<verbose_logs>false</verbose_logs>
</grpc>
```
## Встроенный клиент {#grpc-client}
Можно написать клиент на любом языке программирования, который поддерживается gRPC, с использованием [спецификации](https://github.com/ClickHouse/ClickHouse/blob/master/src/Server/grpc_protos/clickhouse_grpc.proto).
Также можно воспользоваться встроенным Python клиентом. Он расположен в [utils/grpc-client/clickhouse-grpc-client.py](https://github.com/ClickHouse/ClickHouse/blob/master/utils/grpc-client/clickhouse-grpc-client.py) в репозитории. Для работы встроенного клиента требуются Python модули [grpcio и grpcio-tools](https://grpc.io/docs/languages/python/quickstart).
Клиент поддерживает аргументы:
- `--help` – вывести справку и завершить работу.
- `--host HOST, -h HOST` – имя сервера. Значение по умолчанию: `localhost`. Можно задать адрес IPv4 или IPv6.
- `--port PORT` – номер порта. Этот порт должен быть задан в конфигурации сервера ClickHouse настройкой `grpc_port`. Значение по умолчанию: `9100`.
- `--user USER_NAME, -u USER_NAME` – имя пользователя. Значение по умолчанию: `default`.
- `--password PASSWORD` – пароль. Значение по умолчанию: пустая строка.
- `--query QUERY, -q QUERY` – запрос, который выполнится, когда используется неинтерактивный режим работы.
- `--database DATABASE, -d DATABASE` – база данных по умолчанию. Если не указана, то будет использована база данных, заданная в настройках сервера (по умолчанию `default`).
- `--format OUTPUT_FORMAT, -f OUTPUT_FORMAT` – [формат](formats.md) вывода результата. Значение по умолчанию для интерактивного режима: `PrettyCompact`.
- `--debug` – вывод отладочной информации.
Чтобы запустить клиент в интерактивном режиме, не указывайте аргумент `--query`.
В неинтерактивном режиме данные запроса можно передать через `stdin`.
**Пример использования клиента**
В примере создается таблица, и в нее загружаются данные из CSV файла. Затем выводится содержимое таблицы.
``` bash
./clickhouse-grpc-client.py -q "CREATE TABLE grpc_example_table (id UInt32, text String) ENGINE = MergeTree() ORDER BY id;"
echo "0,Input data for" > a.txt ; echo "1,gRPC protocol example" >> a.txt
cat a.txt | ./clickhouse-grpc-client.py -q "INSERT INTO grpc_example_table FORMAT CSV"
./clickhouse-grpc-client.py --format PrettyCompact -q "SELECT * FROM grpc_example_table;"
```
Результат:
``` text
┌─id─┬─text──────────────────┐
│ 0 │ Input data for │
│ 1 │ gRPC protocol example │
└────┴───────────────────────┘
```

7
docs/ru/interfaces/index.md
View File

@ -6,12 +6,13 @@ toc_title: "Введение"
# Интерфейсы {#interfaces} # Интерфейсы {#interfaces}
ClickHouse предоставляет два сетевых интерфейса (оба могут быть дополнительно обернуты в TLS для дополнительной безопасности): ClickHouse предоставляет три сетевых интерфейса (они могут быть обернуты в TLS для дополнительной безопасности):
- [HTTP](http.md), который задокументирован и прост для использования напрямую; - [HTTP](http.md), который задокументирован и прост для использования напрямую;
- [Native TCP](tcp.md), который имеет меньше накладных расходов. - [Native TCP](tcp.md), который имеет меньше накладных расходов;
- [gRPC](grpc.md).
В большинстве случаев рекомендуется использовать подходящий инструмент или библиотеку, а не напрямую взаимодействовать с ClickHouse по сути. Официально поддерживаемые Яндексом: В большинстве случаев рекомендуется использовать подходящий инструмент или библиотеку, а не напрямую взаимодействовать с ClickHouse. Официально поддерживаемые Яндексом:
- [Консольный клиент](cli.md); - [Консольный клиент](cli.md);
- [JDBC-драйвер](jdbc.md); - [JDBC-драйвер](jdbc.md);

57
docs/ru/operations/server-configuration-parameters/settings.md
View File

@ -52,7 +52,7 @@ ClickHouse перезагружает встроенные словари с з
ClickHouse проверяет условия для `min_part_size` и `min_part_size_ratio` и выполнит те блоки `case`, для которых условия совпали. ClickHouse проверяет условия для `min_part_size` и `min_part_size_ratio` и выполнит те блоки `case`, для которых условия совпали.
- Если кусок данных совпадает с условиями, ClickHouse использует указанные метод сжатия. - Если кусок данных совпадает с условиями, ClickHouse использует указанные метод сжатия.
- Если кусок данных совпадает с несколькими блоками `case`, ClickHouse использует перый совпавший блок условий. - Если кусок данных совпадает с несколькими блоками `case`, ClickHouse использует первый совпавший блок условий.
Если ни один `<case>` не подходит, то ClickHouse применит алгоритм сжатия `lz4`. Если ни один `<case>` не подходит, то ClickHouse применит алгоритм сжатия `lz4`.
@ -554,13 +554,13 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
Ключи: Ключи:
- `enabled` – Булевый флаг чтобы включить функциональность, по умолчанию `false`. Установите `true` чтобы разрешить отправку отчетов о сбоях. - `enabled` – Булевый флаг чтобы включить функциональность, по умолчанию `false`. Установите `true` чтобы разрешить отправку отчетов о сбоях.
- `endpoint` – Вы можете переопределить URL на который будут отсылаться отчеты об ошибках и использовать собственную инсталяцию Sentry. Используйте URL синтаксис [Sentry DSN](https://docs.sentry.io/error-reporting/quickstart/?platform=native#configure-the-sdk). - `endpoint` – Вы можете переопределить URL на который будут отсылаться отчеты об ошибках и использовать собственную инсталляцию Sentry. Используйте URL синтаксис [Sentry DSN](https://docs.sentry.io/error-reporting/quickstart/?platform=native#configure-the-sdk).
- `anonymize` - Запретить отсылку имени хоста сервера в отчете о сбое. - `anonymize` - Запретить отсылку имени хоста сервера в отчете о сбое.
- `http_proxy` - Настройка HTTP proxy для отсылки отчетов о сбоях. - `http_proxy` - Настройка HTTP proxy для отсылки отчетов о сбоях.
- `debug` - Настроить клиентскую библиотеку Sentry в debug режим. - `debug` - Настроить клиентскую библиотеку Sentry в debug режим.
- `tmp_path` - Путь в файловой системе для временного хранения состояния отчетов о сбоях перед отправкой на сервер Sentry. - `tmp_path` - Путь в файловой системе для временного хранения состояния отчетов о сбоях перед отправкой на сервер Sentry.
**Рекомендованые настройки** **Рекомендованные настройки**
``` xml ``` xml
<send_crash_reports> <send_crash_reports>
@ -751,9 +751,13 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
## max_thread_pool_size {#max-thread-pool-size} ## max_thread_pool_size {#max-thread-pool-size}
Максимальное количество потоков в глобальном пуле потоков. ClickHouse использует потоки из глобального пула потоков для обработки запросов. Если в пуле нет свободных потоков, то в нем создается еще один. Параметр `max_thread_pool_size` ограничивает максимальное количество потоков в пуле.
Значение по умолчанию: 10000. Возможные значения:
- Положительное целое число.
Значение по умолчанию: `10000`.
**Пример** **Пример**
@ -761,6 +765,38 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
<max_thread_pool_size>12000</max_thread_pool_size> <max_thread_pool_size>12000</max_thread_pool_size>
``` ```
## max_thread_pool_free_size {#max-thread-pool-free-size}
Если в глобальном пуле потоков количество **свободных** потоков больше, чем задано параметром `max_thread_pool_free_size`, то ClickHouse освобождает ресурсы, занятые некоторыми потоками. В таком случае размер пула уменьшается. При необходимости потоки будут созданы заново.
Возможные значения:
- Положительное целое число.
Значение по умолчанию: `1000`.
**Пример**
``` xml
<max_thread_pool_free_size>1200</max_thread_pool_free_size>
```
## thread_pool_queue_size {#thread-pool-queue-size}
Максимальное количество задач, которые запланированы для выполнения в глобальном пуле потоков. При увеличении этого параметра возрастает использование памяти. Рекомендуется, чтобы значение этого параметра совпадало со значением параметра [max_thread_pool_size](#max-thread-pool-size).
Возможные значения:
- Положительное целое число.
Значение по умолчанию: `10000`.
**Пример**
``` xml
<thread_pool_queue_size>12000</thread_pool_queue_size>
```
## merge_tree {#server_configuration_parameters-merge_tree} ## merge_tree {#server_configuration_parameters-merge_tree}
Тонкая настройка таблиц семейства [MergeTree](../../operations/server-configuration-parameters/settings.md). Тонкая настройка таблиц семейства [MergeTree](../../operations/server-configuration-parameters/settings.md).
@ -1011,7 +1047,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
Если таблица не существует, то ClickHouse создаст её. Если структура журнала запросов изменилась при обновлении сервера ClickHouse, то таблица со старой структурой переименовывается, а новая таблица создается автоматически. Если таблица не существует, то ClickHouse создаст её. Если структура журнала запросов изменилась при обновлении сервера ClickHouse, то таблица со старой структурой переименовывается, а новая таблица создается автоматически.
**Example** **Пример**
``` xml ``` xml
<query_views_log> <query_views_log>
@ -1075,9 +1111,8 @@ Parameters:
## query_masking_rules {#query-masking-rules} ## query_masking_rules {#query-masking-rules}
Правила основанные на регурялных выражениях, которые будут применены для всех запросов а также для всех сообщений перед сохранением их в лог на сервере, Правила, основанные на регулярных выражениях, которые будут применены для всех запросов, а также для всех сообщений перед сохранением их в лог на сервере,
`system.query_log`, `system.text_log`, `system.processes` таблицы, а также в логах отсылаемых клиенту. Это позволяет предотвратить `system.query_log`, `system.text_log`, `system.processes` таблицы, а также в логах, отсылаемых клиенту. Это позволяет предотвратить утечку конфиденциальных данных из SQL запросов (такие как имена, электронные письма, личные идентификаторы или номера кредитных карт) в логи.
утечку конфиденциальных данных из SQL запросов (такие как имена, электронные письма, личные идентификаторы или номера кредитных карт) в логи.
**Пример** **Пример**
@ -1096,7 +1131,7 @@ Parameters:
- `regexp` - совместимое с RE2 регулярное выражение (обязательное) - `regexp` - совместимое с RE2 регулярное выражение (обязательное)
- `replace` - строка замены для конфиденциальных данных (опционально, по умолчанию - шесть звездочек) - `replace` - строка замены для конфиденциальных данных (опционально, по умолчанию - шесть звездочек)
Правила маскировки применяются ко всему запросу (для предотвращения утечки конфиденциальных данных из неправильно оформленных / не интерпритируемых запросов). Правила маскировки применяются ко всему запросу (для предотвращения утечки конфиденциальных данных из неправильно оформленных / не интерпретируемых запросов).
`system.events` таблица содержит счетчик `QueryMaskingRulesMatch` который считает общее кол-во совпадений правил маскировки. `system.events` таблица содержит счетчик `QueryMaskingRulesMatch` который считает общее кол-во совпадений правил маскировки.
@ -1418,7 +1453,7 @@ ClickHouse использует ZooKeeper для хранения метадан
Также вы можете добавить секции `memory` — означает хранение информации только в памяти, без записи на диск, и `ldap` — означает хранения информации на [LDAP-сервере](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol). Также вы можете добавить секции `memory` — означает хранение информации только в памяти, без записи на диск, и `ldap` — означает хранения информации на [LDAP-сервере](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol).
Чтобы добавить LDAP-сервер в качестве удаленного каталога пользователей, которые не определены локально, определите один раздел `ldap` со следующими параметрами: Чтобы добавить LDAP-сервер в качестве удаленного каталога пользователей, которые не определены локально, определите один раздел `ldap` со следующими параметрами:
- `server` — имя одного из LDAP-серверов, определенных в секции `ldap_servers` конфигурациионного файла. Этот параметр явялется необязательным и может быть пустым. - `server` — имя одного из LDAP-серверов, определенных в секции `ldap_servers` конфигурационного файла. Этот параметр является необязательным и может быть пустым.
- `roles` — раздел со списком локально определенных ролей, которые будут назначены каждому пользователю, полученному с LDAP-сервера. Если роли не заданы, пользователь не сможет выполнять никаких действий после аутентификации. Если какая-либо из перечисленных ролей не определена локально во время проверки подлинности, попытка проверки подлинности завершится неудачей, как если бы предоставленный пароль был неверным. - `roles` — раздел со списком локально определенных ролей, которые будут назначены каждому пользователю, полученному с LDAP-сервера. Если роли не заданы, пользователь не сможет выполнять никаких действий после аутентификации. Если какая-либо из перечисленных ролей не определена локально во время проверки подлинности, попытка проверки подлинности завершится неудачей, как если бы предоставленный пароль был неверным.
**Пример** **Пример**

20
docs/ru/operations/settings/merge-tree-settings.md
View File

@ -355,3 +355,23 @@ Eсли суммарное число активных кусков во все
- 1 — куски данных открепляются. - 1 — куски данных открепляются.
Значение по умолчанию: `0`. Значение по умолчанию: `0`.
## merge_tree_clear_old_temporary_directories_interval_seconds {#setting-merge-tree-clear-old-temporary-directories-interval-seconds}
Задает интервал в секундах для удаления старых временных каталогов на сервере ClickHouse.
Возможные значения:
- Положительное целое число.
Значение по умолчанию: `60` секунд.
## merge_tree_clear_old_parts_interval_seconds {#setting-merge-tree-clear-old-parts-interval-seconds}
Задает интервал в секундах для удаления старых кусков данных, журналов предзаписи (WAL) и мутаций на сервере ClickHouse.
Возможные значения:
- Положительное целое число.
Значение по умолчанию: `1` секунда.

20
docs/ru/operations/settings/settings.md
View File

@ -807,26 +807,6 @@ ClickHouse может парсить только базовый формат `Y
Значение по умолчанию: 2013265920. Значение по умолчанию: 2013265920.
## merge_tree_clear_old_temporary_directories_interval_seconds {#setting-merge-tree-clear-old-temporary-directories-interval-seconds}
Задает интервал в секундах для удаления старых временных каталогов на сервере ClickHouse.
Возможные значения:
- Положительное целое число.
Значение по умолчанию: `60` секунд.
## merge_tree_clear_old_parts_interval_seconds {#setting-merge-tree-clear-old-parts-interval-seconds}
Задает интервал в секундах для удаления старых кусков данных, журналов предзаписи (WAL) и мутаций на сервере ClickHouse .
Возможные значения:
- Положительное целое число.
Значение по умолчанию: `1` секунда.
## min_bytes_to_use_direct_io {#settings-min-bytes-to-use-direct-io} ## min_bytes_to_use_direct_io {#settings-min-bytes-to-use-direct-io}
Минимальный объём данных, необходимый для прямого (небуферизованного) чтения/записи (direct I/O) на диск. Минимальный объём данных, необходимый для прямого (небуферизованного) чтения/записи (direct I/O) на диск.

66
docs/ru/sql-reference/aggregate-functions/reference/sparkbar.md Normal file
View File

@ -0,0 +1,66 @@
---
toc_priority: 311
toc_title: sparkbar
---
# sparkbar {#sparkbar}
Функция строит гистограмму частот по заданным значениям `x` и частоте повторения этих значений `y` на интервале `[min_x, max_x]`.
Если интервал для построения не указан, то в качестве нижней границы интервала будет взято минимальное значение `x`, а в качестве верхней границы — максимальное значение `x`.
**Синтаксис**
``` sql
sparkbar(width[, min_x, max_x])(x, y)
```
**Параметры**
- `width` — Количество столбцов гистограммы. Тип: [Integer](../../../sql-reference/data-types/int-uint.md).
- `min_x` — Начало интервала. Необязательный параметр.
- `max_x` — Конец интервала. Необязательный параметр.
**Аргументы**
- `x` — Поле со значениями.
- `y` — Поле с частотой повторения значений.
**Возвращаемые значения**
- Гистограмма частот.
**Пример**
Запрос:
``` sql
CREATE TABLE spark_bar_data (`cnt` UInt64,`event_date` Date) ENGINE = MergeTree ORDER BY event_date SETTINGS index_granularity = 8192;
INSERT INTO spark_bar_data VALUES(1,'2020-01-01'),(4,'2020-01-02'),(5,'2020-01-03'),(2,'2020-01-04'),(3,'2020-01-05'),(7,'2020-01-06'),(6,'2020-01-07'),(8,'2020-01-08'),(2,'2020-01-11');
SELECT sparkbar(9)(event_date,cnt) FROM spark_bar_data;
SELECT sparkbar(9,toDate('2020-01-01'),toDate('2020-01-10'))(event_date,cnt) FROM spark_bar_data;
```
Результат:
``` text
┌─sparkbar(9)(event_date, cnt)─┐
│ │
│ ▁▅▄▃██▅ ▁ │
│ │
└──────────────────────────────┘
┌─sparkbar(9, toDate('2020-01-01'), toDate('2020-01-10'))(event_date, cnt)─┐
│ │
│▁▄▄▂▅▇█▁ │
│ │
└──────────────────────────────────────────────────────────────────────────┘
```

2
docs/ru/sql-reference/statements/select/order-by.md
View File

@ -271,7 +271,7 @@ SELECT * FROM collate_test ORDER BY s ASC COLLATE 'en';
## Модификатор ORDER BY expr WITH FILL {#orderby-with-fill} ## Модификатор ORDER BY expr WITH FILL {#orderby-with-fill}
Этот модификатор также может быть скобинирован с модификатором [LIMIT ... WITH TIES](../../../sql-reference/statements/select/limit.md#limit-with-ties) Этот модификатор также может быть скомбинирован с модификатором [LIMIT ... WITH TIES](../../../sql-reference/statements/select/limit.md#limit-with-ties)
Модификатор `WITH FILL` может быть установлен после `ORDER BY expr` с опциональными параметрами `FROM expr`, `TO expr` и `STEP expr`. Модификатор `WITH FILL` может быть установлен после `ORDER BY expr` с опциональными параметрами `FROM expr`, `TO expr` и `STEP expr`.
Все пропущенные значения для колонки `expr` будут заполнены значениями, соответствующими предполагаемой последовательности значений колонки, другие колонки будут заполнены значениями по умолчанию. Все пропущенные значения для колонки `expr` будут заполнены значениями, соответствующими предполагаемой последовательности значений колонки, другие колонки будут заполнены значениями по умолчанию.

112
docs/zh/sql-reference/functions/time-window-functions.md Normal file
View File

@ -0,0 +1,112 @@
---
toc_priority: 68
toc_title: 时间窗口
---
# 时间窗口函数 {#time-window-han-shu}
时间窗口函数用于获取窗口的起始(包含边界)和结束时间(不包含边界)。系统支持的时间窗口函数如下：
## tumble {#time-window-functions-tumble}
tumble窗口是连续的、不重叠的固定大小(`interval`)时间窗口。
``` sql
tumble(time_attr, interval [, timezone])
```
**参数**
- `time_attr` - [DateTime](../../sql-reference/data-types/datetime.md)类型的时间数据。
- `interval` - [Interval](../../sql-reference/data-types/special-data-types/interval.md)类型的窗口大小。
- `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) 类型的时区(可选参数).
**返回值**
- tumble窗口的开始(包含边界)和结束时间(不包含边界)
类型: `Tuple(DateTime, DateTime)`
**示例**
查询：
``` sql
SELECT tumble(now(), toIntervalDay('1'))
```
结果：
``` text
┌─tumble(now(), toIntervalDay('1'))─────────────┐
│ ['2020-01-01 00:00:00','2020-01-02 00:00:00'] │
└───────────────────────────────────────────────┘
```
## hop {#time-window-functions-hop}
hop窗口是一个固定大小(`window_interval`)的时间窗口，并按照一个固定的滑动间隔(`hop_interval`)滑动。当滑动间隔小于窗口大小时，滑动窗口间存在重叠，此时一个数据可能存在于多个窗口。
``` sql
hop(time_attr, hop_interval, window_interval [, timezone])
```
**参数**
- `time_attr` - [DateTime](../../sql-reference/data-types/datetime.md)类型的时间数据。
- `hop_interval` - [Interval](../../sql-reference/data-types/special-data-types/interval.md)类型的滑动间隔，需要大于0。
- `window_interval` - [Interval](../../sql-reference/data-types/special-data-types/interval.md)类型的窗口大小，需要大于0。
- `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) 类型的时区(可选参数)。
**返回值**
- hop窗口的开始(包含边界)和结束时间(不包含边界)。由于一个数据可能存在于多个窗口，脱离window view单独调用该函数时只返回第一个窗口数据。
类型: `Tuple(DateTime, DateTime)`
**示例**
查询：
``` sql
SELECT hop(now(), INTERVAL '1' SECOND, INTERVAL '2' SECOND)
```
结果：
``` text
┌─hop(now(), toIntervalSecond('1'), toIntervalSecond('2'))──┐
│ ('2020-01-14 16:58:22','2020-01-14 16:58:24') │
└───────────────────────────────────────────────────────────┘
```
## tumbleStart {#time-window-functions-tumblestart}
返回tumble窗口的开始时间(包含边界)。
``` sql
tumbleStart(time_attr, interval [, timezone]);
```
## tumbleEnd {#time-window-functions-tumbleend}
返回tumble窗口的结束时间(不包含边界)。
``` sql
tumbleEnd(time_attr, interval [, timezone]);
```
## hopStart {#time-window-functions-hopstart}
返回hop窗口的开始时间(包含边界)。
``` sql
hopStart(time_attr, hop_interval, window_interval [, timezone]);
```
## hopEnd {#time-window-functions-hopend}
返回hop窗口的结束时间(不包含边界)。
``` sql
hopEnd(time_attr, hop_interval, window_interval [, timezone]);
```

119
docs/zh/sql-reference/statements/create/view.md
View File

@ -5,7 +5,7 @@ toc_title: VIEW
# CREATE VIEW {#create-view} # CREATE VIEW {#create-view}
创建一个新视图。 有两种类型的视图：普通视图和物化视图。 创建一个新视图。 有两种类型的视图：普通视图，物化视图，Live视图和Window视图。
## Normal {#normal} ## Normal {#normal}
@ -241,3 +241,120 @@ Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table defa
- 使用定期刷新从系统表中查看指标。 - 使用定期刷新从系统表中查看指标。
[原始文章](https://clickhouse.com/docs/en/sql-reference/statements/create/view/) <!--hide--> [原始文章](https://clickhouse.com/docs/en/sql-reference/statements/create/view/) <!--hide-->
## Window View [Experimental] {#window-view}
!!! important "重要"
这是一项试验性功能，可能会在未来版本中以向后不兼容的方式进行更改。
通过[allow_experimental_window_view](../../../operations/settings/settings.md#allow-experimental-window-view)启用window view以及`WATCH`语句。输入命令
`set allow_experimental_window_view = 1`。
``` sql
CREATE WINDOW VIEW [IF NOT EXISTS] [db.]table_name [TO [db.]table_name] [ENGINE = engine] [WATERMARK = strategy] [ALLOWED_LATENESS = interval_function] AS SELECT ... GROUP BY time_window_function
```
Window view可以通过时间窗口聚合数据，并在满足窗口触发条件时自动触发对应窗口计算。其通过将计算状态保存降低处理延迟，支持将处理结果输出至目标表或通过`WATCH`语句输出至终端。
创建window view的方式和创建物化视图类似。Window view使用默认为`AggregatingMergeTree`的内部存储引擎存储计算中间状态。
### 时间窗口函数 {#window-view-shi-jian-chuang-kou-han-shu}
[时间窗口函数](../../functions/time-window-functions.md)用于获取窗口的起始和结束时间。Window view需要和时间窗口函数配合使用。
### 时间属性 {#window-view-shi-jian-shu-xing}
Window view 支持**处理时间**和**事件时间**两种时间类型。
**处理时间**为默认时间类型，该模式下window view使用本地机器时间计算窗口数据。“处理时间”时间类型计算简单，但具有不确定性。该模式下时间可以为时间窗口函数的第一个参数`time_attr`，或通过函数`now()`使用当前机器时间。下面的例子展示了使用“处理时间”创建window view的例子。
``` sql
CREATE WINDOW VIEW wv AS SELECT count(number), tumbleStart(w_id) as w_start from date GROUP BY tumble(now(), INTERVAL '5' SECOND) as w_id
```
**事件时间** 是事件真实发生的时间，该时间往往在事件发生时便嵌入数据记录。事件时间处理提供较高的确定性，可以处理乱序数据以及迟到数据。Window view通过水位线(`WATERMARK`)启用事件时间处理。
Window view提供如下三种水位线策略：
* `STRICTLY_ASCENDING`: 提交观测到的最大时间作为水位线，小于最大观测时间的数据不算迟到。
* `ASCENDING`: 提交观测到的最大时间减1作为水位线。小于或等于最大观测时间的数据不算迟到。
* `BOUNDED`: WATERMARK=INTERVAL. 提交最大观测时间减去固定间隔(`INTERVAL`)做为水位线。
以下为使用`WATERMARK`创建window view的示例：
``` sql
CREATE WINDOW VIEW wv WATERMARK=STRICTLY_ASCENDING AS SELECT count(number) FROM date GROUP BY tumble(timestamp, INTERVAL '5' SECOND);
CREATE WINDOW VIEW wv WATERMARK=ASCENDING AS SELECT count(number) FROM date GROUP BY tumble(timestamp, INTERVAL '5' SECOND);
CREATE WINDOW VIEW wv WATERMARK=INTERVAL '3' SECOND AS SELECT count(number) FROM date GROUP BY tumble(timestamp, INTERVAL '5' SECOND);
```
通常，窗口会在水位线到达时触发，水位线到达之后的数据会被丢弃。Window view可以通过设置`ALLOWED_LATENESS=INTERVAL`来开启迟到消息处理。示例如下：
``` sql
CREATE WINDOW VIEW test.wv TO test.dst WATERMARK=ASCENDING ALLOWED_LATENESS=INTERVAL '2' SECOND AS SELECT count(a) AS count, tumbleEnd(wid) AS w_end FROM test.mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND) AS wid;
```
需要注意的是，迟到消息需要更新之前的处理结果。与在窗口结束时触发不同，迟到消息到达时window view会立即触发计算。因此，会导致同一个窗口输出多次计算结果。用户需要注意这种情况，并消除重复结果。
### 新窗口监控 {#window-view-xin-chuang-kou-jian-kong}
Window view可以通过`WATCH`语句将处理结果推送至终端，或通过`TO`语句将结果推送至数据表。
``` sql
WATCH [db.]name [LIMIT n]
```
`WATCH`语句和`LIVE VIEW`中的类似。支持设置`LIMIT`参数，输出消息数目达到`LIMIT`限制时结束查询。
### 设置 {#window-view-she-zhi}
- `window_view_clean_interval`: window view清除过期数据间隔(单位为秒)。系统会定期清除过期数据，尚未触发的窗口数据不会被清除。
- `window_view_heartbeat_interval`: 用于判断watch查询活跃的心跳时间间隔。
### 示例 {#window-view-shi-li}
假设我们需要每10秒统计一次`data`表中的点击日志，且`data`表的结构如下：
``` sql
CREATE TABLE data ( `id` UInt64, `timestamp` DateTime) ENGINE = Memory;
```
首先，使用10秒大小的tumble函数创建window view。
``` sql
CREATE WINDOW VIEW wv as select count(id), tumbleStart(w_id) as window_start from data group by tumble(timestamp, INTERVAL '10' SECOND) as w_id
```
随后，我们使用`WATCH`语句获取计算结果。
``` sql
WATCH wv
```
当日志插入表`data`时，
``` sql
INSERT INTO data VALUES(1,now())
```
`WATCH`语句会输出如下结果：
``` text
┌─count(id)─┬────────window_start─┐
│ 1 │ 2020-01-14 16:56:40 │
└───────────┴─────────────────────┘
```
或者，我们可以通过`TO`关键字将处理结果输出至另一张表。
``` sql
CREATE WINDOW VIEW wv TO dst AS SELECT count(id), tumbleStart(w_id) as window_start FROM data GROUP BY tumble(timestamp, INTERVAL '10' SECOND) as w_id
```
ClickHouse测试中提供了更多的示例(以`*window_view*`命名)。
### Window View 使用场景 {#window-view-shi-yong-chang-jing}
Window view 在以下场景有用：
* **监控**: 以时间维度聚合及处理数据，并将处理结果输出至目标表。用户可通过目标表获取并操作计算结果。
* **分析**: 以时间维度进行数据分析. 当数据源非常庞大时，window view可以减少重复全表查询的计算量。

16
programs/client/Client.cpp
View File

@ -20,9 +20,7 @@
#include <base/argsToConfig.h> #include <base/argsToConfig.h>
#include <base/find_symbols.h> #include <base/find_symbols.h>
#if !defined(ARCADIA_BUILD) #include <Common/config_version.h>
# include <Common/config_version.h>
#endif
#include <Common/Exception.h> #include <Common/Exception.h>
#include <Common/formatReadable.h> #include <Common/formatReadable.h>
#include <Common/TerminalSize.h> #include <Common/TerminalSize.h>
@ -705,6 +703,12 @@ bool Client::processWithFuzzing(const String & full_query)
throw; throw;
} }
if (!orig_ast)
{
// Can't continue after a parsing error
return true;
}
// `USE db` should not be executed // `USE db` should not be executed
// since this will break every query after `DROP db` // since this will break every query after `DROP db`
if (orig_ast->as<ASTUseQuery>()) if (orig_ast->as<ASTUseQuery>())
@ -712,12 +716,6 @@ bool Client::processWithFuzzing(const String & full_query)
return true; return true;
} }
if (!orig_ast)
{
// Can't continue after a parsing error
return true;
}
// Don't repeat: // Don't repeat:
// - INSERT -- Because the tables may grow too big. // - INSERT -- Because the tables may grow too big.
// - CREATE -- Because first we run the unmodified query, it will succeed, // - CREATE -- Because first we run the unmodified query, it will succeed,

6
programs/keeper/Keeper.cpp
View File

@ -22,10 +22,8 @@
#include <pwd.h> #include <pwd.h>
#include <Coordination/FourLetterCommand.h> #include <Coordination/FourLetterCommand.h>
#if !defined(ARCADIA_BUILD) #include "config_core.h"
# include "config_core.h" #include "Common/config_version.h"
# include "Common/config_version.h"
#endif
#if USE_SSL #if USE_SSL
# include <Poco/Net/Context.h> # include <Poco/Net/Context.h>

10
programs/local/LocalServer.cpp
View File

@ -388,12 +388,6 @@ void LocalServer::setupUsers()
} }
String LocalServer::getQueryTextPrefix()
{
return getInitialCreateTableQuery();
}
void LocalServer::connect() void LocalServer::connect()
{ {
connection_parameters = ConnectionParameters(config()); connection_parameters = ConnectionParameters(config());
@ -463,6 +457,10 @@ try
} }
#endif #endif
String initial_query = getInitialCreateTableQuery();
if (!initial_query.empty())
processQueryText(initial_query);
if (is_interactive && !delayed_interactive) if (is_interactive && !delayed_interactive)
{ {
runInteractive(); runInteractive();

1
programs/local/LocalServer.h
View File

@ -37,7 +37,6 @@ protected:
void processError(const String & query) const override; void processError(const String & query) const override;
String getName() const override { return "local"; } String getName() const override { return "local"; }
String getQueryTextPrefix() override;
void printHelpMessage(const OptionsDescription & options_description) override; void printHelpMessage(const OptionsDescription & options_description) override;
void addOptions(OptionsDescription & options_description) override; void addOptions(OptionsDescription & options_description) override;

4
programs/main.cpp
View File

@ -13,9 +13,7 @@
#include <tuple> #include <tuple>
#include <utility> /// pair #include <utility> /// pair
#if !defined(ARCADIA_BUILD) #include "config_tools.h"
# include "config_tools.h"
#endif
#include <Common/StringUtils/StringUtils.h> #include <Common/StringUtils/StringUtils.h>
#include <Common/getHashOfLoadedBinary.h> #include <Common/getHashOfLoadedBinary.h>

8
programs/server/Server.cpp
View File

@ -82,10 +82,8 @@
#include <Compression/CompressionCodecEncrypted.h> #include <Compression/CompressionCodecEncrypted.h>
#include <filesystem> #include <filesystem>
#if !defined(ARCADIA_BUILD) #include "config_core.h"
# include "config_core.h" #include "Common/config_version.h"
# include "Common/config_version.h"
#endif
#if defined(OS_LINUX) #if defined(OS_LINUX)
# include <sys/mman.h> # include <sys/mman.h>
@ -96,7 +94,7 @@
#endif #endif
#if USE_SSL #if USE_SSL
# if USE_INTERNAL_SSL_LIBRARY && !defined(ARCADIA_BUILD) # if USE_INTERNAL_SSL_LIBRARY
# include <Compression/CompressionCodecEncrypted.h> # include <Compression/CompressionCodecEncrypted.h>
# endif # endif
# include <Poco/Net/Context.h> # include <Poco/Net/Context.h>

32
programs/server/config.xml
View File

@ -650,6 +650,38 @@
</replica> </replica>
</shard> </shard>
</test_shard_localhost> </test_shard_localhost>
<test_cluster_one_shard_three_replicas_localhost>
<shard>
<internal_replication>false</internal_replication>
<replica>
<host>127.0.0.1</host>
<port>9000</port>
</replica>
<replica>
<host>127.0.0.2</host>
<port>9000</port>
</replica>
<replica>
<host>127.0.0.3</host>
<port>9000</port>
</replica>
</shard>
<!--shard>
<internal_replication>false</internal_replication>
<replica>
<host>127.0.0.1</host>
<port>9000</port>
</replica>
<replica>
<host>127.0.0.2</host>
<port>9000</port>
</replica>
<replica>
<host>127.0.0.3</host>
<port>9000</port>
</replica>
</shard-->
</test_cluster_one_shard_three_replicas_localhost>
<test_cluster_two_shards_localhost> <test_cluster_two_shards_localhost>
<shard> <shard>
<replica> <replica>

134
src/Access/EnabledQuota.cpp
View File

@ -2,6 +2,7 @@
#include <Access/QuotaUsage.h> #include <Access/QuotaUsage.h>
#include <Common/Exception.h> #include <Common/Exception.h>
#include <Common/quoteString.h> #include <Common/quoteString.h>
#include <Common/thread_local_rng.h>
#include <base/chrono_io.h> #include <base/chrono_io.h>
#include <base/range.h> #include <base/range.h>
#include <boost/smart_ptr/make_shared.hpp> #include <boost/smart_ptr/make_shared.hpp>
@ -15,6 +16,7 @@ namespace ErrorCodes
extern const int QUOTA_EXPIRED; extern const int QUOTA_EXPIRED;
} }
struct EnabledQuota::Impl struct EnabledQuota::Impl
{ {
[[noreturn]] static void throwQuotaExceed( [[noreturn]] static void throwQuotaExceed(
@ -35,54 +37,6 @@ struct EnabledQuota::Impl
} }
/// Returns the end of the current interval. If the passed `current_time` is greater than that end,
/// the function automatically recalculates the interval's end by adding the interval's duration
/// one or more times until the interval's end is greater than `current_time`.
/// If that recalculation occurs the function also resets amounts of resources used and sets the variable
/// `counters_were_reset`.
static std::chrono::system_clock::time_point getEndOfInterval(
const Interval & interval, std::chrono::system_clock::time_point current_time, bool & counters_were_reset)
{
auto & end_of_interval = interval.end_of_interval;
auto end_loaded = end_of_interval.load();
auto end = std::chrono::system_clock::time_point{end_loaded};
if (current_time < end)
{
counters_were_reset = false;
return end;
}
bool need_reset_counters = false;
do
{
/// Calculate the end of the next interval:
/// | X |
/// end current_time next_end = end + duration * n
/// where n is an integer number, n >= 1.
const auto duration = interval.duration;
UInt64 n = static_cast<UInt64>((current_time - end + duration) / duration);
end = end + duration * n;
if (end_of_interval.compare_exchange_strong(end_loaded, end.time_since_epoch()))
{
/// We reset counters only if the interval's end has been calculated before.
/// If it hasn't we just calculate the interval's end for the first time and don't reset counters yet.
need_reset_counters = (end_loaded.count() != 0);
break;
}
end = std::chrono::system_clock::time_point{end_loaded};
}
while (current_time >= end);
if (need_reset_counters)
{
boost::range::fill(interval.used, 0);
counters_were_reset = true;
}
return end;
}
static void used( static void used(
const String & user_name, const String & user_name,
const Intervals & intervals, const Intervals & intervals,
@ -91,24 +45,22 @@ struct EnabledQuota::Impl
std::chrono::system_clock::time_point current_time, std::chrono::system_clock::time_point current_time,
bool check_exceeded) bool check_exceeded)
{ {
auto quota_type_i = static_cast<size_t>(quota_type);
for (const auto & interval : intervals.intervals) for (const auto & interval : intervals.intervals)
{ {
auto quota_type_i = static_cast<size_t>(quota_type);
QuotaValue used = (interval.used[quota_type_i] += value); QuotaValue used = (interval.used[quota_type_i] += value);
QuotaValue max = interval.max[quota_type_i]; QuotaValue max = interval.max[quota_type_i];
if (!max) if (!max)
continue; continue;
if (used > max) if (used > max)
{ {
bool counters_were_reset = false; bool counters_were_reset = false;
auto end_of_interval = getEndOfInterval(interval, current_time, counters_were_reset); auto end_of_interval = interval.getEndOfInterval(current_time, counters_were_reset);
if (counters_were_reset) if (counters_were_reset)
{
used = (interval.used[quota_type_i] += value); used = (interval.used[quota_type_i] += value);
if ((used > max) && check_exceeded)
throwQuotaExceed(user_name, intervals.quota_name, quota_type, used, max, interval.duration, end_of_interval); if (check_exceeded && (used > max))
}
else if (check_exceeded)
throwQuotaExceed(user_name, intervals.quota_name, quota_type, used, max, interval.duration, end_of_interval); throwQuotaExceed(user_name, intervals.quota_name, quota_type, used, max, interval.duration, end_of_interval);
} }
} }
@ -127,10 +79,11 @@ struct EnabledQuota::Impl
QuotaValue max = interval.max[quota_type_i]; QuotaValue max = interval.max[quota_type_i];
if (!max) if (!max)
continue; continue;
if (used > max) if (used > max)
{ {
bool counters_were_reset = false; bool counters_were_reset = false;
std::chrono::system_clock::time_point end_of_interval = getEndOfInterval(interval, current_time, counters_were_reset); auto end_of_interval = interval.getEndOfInterval(current_time, counters_were_reset);
if (!counters_were_reset) if (!counters_were_reset)
throwQuotaExceed(user_name, intervals.quota_name, quota_type, used, max, interval.duration, end_of_interval); throwQuotaExceed(user_name, intervals.quota_name, quota_type, used, max, interval.duration, end_of_interval);
} }
@ -145,17 +98,32 @@ struct EnabledQuota::Impl
for (auto quota_type : collections::range(QuotaType::MAX)) for (auto quota_type : collections::range(QuotaType::MAX))
checkExceeded(user_name, intervals, quota_type, current_time); checkExceeded(user_name, intervals, quota_type, current_time);
} }
static std::chrono::system_clock::duration randomDuration(std::chrono::seconds max)
{
auto count = std::chrono::duration_cast<std::chrono::system_clock::duration>(max).count();
std::uniform_int_distribution<Int64> distribution{0, count - 1};
return std::chrono::system_clock::duration(distribution(thread_local_rng));
}
}; };
EnabledQuota::Interval::Interval() EnabledQuota::Interval::Interval(std::chrono::seconds duration_, bool randomize_interval_, std::chrono::system_clock::time_point current_time_)
: duration(duration_) , randomize_interval(randomize_interval_)
{ {
std::chrono::system_clock::time_point initial_end{};
if (randomize_interval_)
initial_end += Impl::randomDuration(duration_);
end_of_interval = initial_end.time_since_epoch();
for (auto quota_type : collections::range(QuotaType::MAX)) for (auto quota_type : collections::range(QuotaType::MAX))
{ {
auto quota_type_i = static_cast<size_t>(quota_type); auto quota_type_i = static_cast<size_t>(quota_type);
used[quota_type_i].store(0); used[quota_type_i].store(0);
max[quota_type_i] = 0; max[quota_type_i] = 0;
} }
getEndOfInterval(current_time_); /// Force updating the end of the interval for the first time.
} }
@ -177,6 +145,55 @@ EnabledQuota::Interval & EnabledQuota::Interval::operator =(const Interval & src
} }
/// Returns the end of the current interval. If the passed `current_time` is greater than that end,
/// the function automatically recalculates the interval's end by adding the interval's duration
/// one or more times until the interval's end is greater than `current_time`.
/// If that recalculation occurs the function also resets amounts of resources used and sets the variable
/// `counters_were_reset`.
std::chrono::system_clock::time_point EnabledQuota::Interval::getEndOfInterval(std::chrono::system_clock::time_point current_time) const
{
bool counters_were_reset;
return getEndOfInterval(current_time, counters_were_reset);
}
std::chrono::system_clock::time_point EnabledQuota::Interval::getEndOfInterval(std::chrono::system_clock::time_point current_time, bool & counters_were_reset) const
{
auto end_loaded = end_of_interval.load();
auto end = std::chrono::system_clock::time_point{end_loaded};
if (current_time < end)
{
counters_were_reset = false;
return end;
}
bool need_reset_counters = false;
do
{
/// Calculate the end of the next interval:
/// | X |
/// end current_time next_end = end + duration * n
/// where n is an integer number, n >= 1.
UInt64 n = static_cast<UInt64>((current_time - end + duration) / duration);
end = end + duration * n;
if (end_of_interval.compare_exchange_strong(end_loaded, end.time_since_epoch()))
{
need_reset_counters = true;
break;
}
end = std::chrono::system_clock::time_point{end_loaded};
}
while (current_time >= end);
if (need_reset_counters)
{
boost::range::fill(used, 0);
counters_were_reset = true;
}
return end;
}
std::optional<QuotaUsage> EnabledQuota::Intervals::getUsage(std::chrono::system_clock::time_point current_time) const std::optional<QuotaUsage> EnabledQuota::Intervals::getUsage(std::chrono::system_clock::time_point current_time) const
{ {
if (!quota_id) if (!quota_id)
@ -192,8 +209,7 @@ std::optional<QuotaUsage> EnabledQuota::Intervals::getUsage(std::chrono::system_
auto & out = usage.intervals.back(); auto & out = usage.intervals.back();
out.duration = in.duration; out.duration = in.duration;
out.randomize_interval = in.randomize_interval; out.randomize_interval = in.randomize_interval;
bool counters_were_reset = false; out.end_of_interval = in.getEndOfInterval(current_time);
out.end_of_interval = Impl::getEndOfInterval(in, current_time, counters_were_reset);
for (auto quota_type : collections::range(QuotaType::MAX)) for (auto quota_type : collections::range(QuotaType::MAX))
{ {
auto quota_type_i = static_cast<size_t>(quota_type); auto quota_type_i = static_cast<size_t>(quota_type);

6
src/Access/EnabledQuota.h
View File

@ -73,9 +73,13 @@ private:
bool randomize_interval = false; bool randomize_interval = false;
mutable std::atomic<std::chrono::system_clock::duration> end_of_interval; mutable std::atomic<std::chrono::system_clock::duration> end_of_interval;
Interval(); Interval(std::chrono::seconds duration_, bool randomize_interval_, std::chrono::system_clock::time_point current_time_);
Interval(const Interval & src) { *this = src; } Interval(const Interval & src) { *this = src; }
Interval & operator =(const Interval & src); Interval & operator =(const Interval & src);
std::chrono::system_clock::time_point getEndOfInterval(std::chrono::system_clock::time_point current_time) const;
std::chrono::system_clock::time_point getEndOfInterval(std::chrono::system_clock::time_point current_time, bool & counters_were_reset) const;
}; };
struct Intervals struct Intervals

29
src/Access/QuotaCache.cpp
View File

@ -4,7 +4,6 @@
#include <Access/QuotaUsage.h> #include <Access/QuotaUsage.h>
#include <Access/AccessControl.h> #include <Access/AccessControl.h>
#include <Common/Exception.h> #include <Common/Exception.h>
#include <Common/thread_local_rng.h>
#include <base/range.h> #include <base/range.h>
#include <boost/range/adaptor/map.hpp> #include <boost/range/adaptor/map.hpp>
#include <boost/range/algorithm/copy.hpp> #include <boost/range/algorithm/copy.hpp>
@ -22,17 +21,6 @@ namespace ErrorCodes
} }
namespace
{
std::chrono::system_clock::duration randomDuration(std::chrono::seconds max)
{
auto count = std::chrono::duration_cast<std::chrono::system_clock::duration>(max).count();
std::uniform_int_distribution<Int64> distribution{0, count - 1};
return std::chrono::system_clock::duration(distribution(thread_local_rng));
}
}
void QuotaCache::QuotaInfo::setQuota(const QuotaPtr & quota_, const UUID & quota_id_) void QuotaCache::QuotaInfo::setQuota(const QuotaPtr & quota_, const UUID & quota_id_)
{ {
quota = quota_; quota = quota_;
@ -94,18 +82,21 @@ boost::shared_ptr<const EnabledQuota::Intervals> QuotaCache::QuotaInfo::getOrBui
auto it = key_to_intervals.find(key); auto it = key_to_intervals.find(key);
if (it != key_to_intervals.end()) if (it != key_to_intervals.end())
return it->second; return it->second;
return rebuildIntervals(key); return rebuildIntervals(key, std::chrono::system_clock::now());
} }
void QuotaCache::QuotaInfo::rebuildAllIntervals() void QuotaCache::QuotaInfo::rebuildAllIntervals()
{ {
if (key_to_intervals.empty())
return;
auto current_time = std::chrono::system_clock::now();
for (const String & key : key_to_intervals | boost::adaptors::map_keys) for (const String & key : key_to_intervals | boost::adaptors::map_keys)
rebuildIntervals(key); rebuildIntervals(key, current_time);
} }
boost::shared_ptr<const EnabledQuota::Intervals> QuotaCache::QuotaInfo::rebuildIntervals(const String & key) boost::shared_ptr<const EnabledQuota::Intervals> QuotaCache::QuotaInfo::rebuildIntervals(const String & key, std::chrono::system_clock::time_point current_time)
{ {
auto new_intervals = boost::make_shared<Intervals>(); auto new_intervals = boost::make_shared<Intervals>();
new_intervals->quota_name = quota->getName(); new_intervals->quota_name = quota->getName();
@ -115,14 +106,8 @@ boost::shared_ptr<const EnabledQuota::Intervals> QuotaCache::QuotaInfo::rebuildI
intervals.reserve(quota->all_limits.size()); intervals.reserve(quota->all_limits.size());
for (const auto & limits : quota->all_limits) for (const auto & limits : quota->all_limits)
{ {
intervals.emplace_back(); intervals.emplace_back(limits.duration, limits.randomize_interval, current_time);
auto & interval = intervals.back(); auto & interval = intervals.back();
interval.duration = limits.duration;
std::chrono::system_clock::time_point end_of_interval{};
interval.randomize_interval = limits.randomize_interval;
if (limits.randomize_interval)
end_of_interval += randomDuration(limits.duration);
interval.end_of_interval = end_of_interval.time_since_epoch();
for (auto quota_type : collections::range(QuotaType::MAX)) for (auto quota_type : collections::range(QuotaType::MAX))
{ {
auto quota_type_i = static_cast<size_t>(quota_type); auto quota_type_i = static_cast<size_t>(quota_type);

2
src/Access/QuotaCache.h
View File

@ -43,7 +43,7 @@ private:
String calculateKey(const EnabledQuota & enabled_quota) const; String calculateKey(const EnabledQuota & enabled_quota) const;
boost::shared_ptr<const Intervals> getOrBuildIntervals(const String & key); boost::shared_ptr<const Intervals> getOrBuildIntervals(const String & key);
boost::shared_ptr<const Intervals> rebuildIntervals(const String & key); boost::shared_ptr<const Intervals> rebuildIntervals(const String & key, std::chrono::system_clock::time_point current_time);
void rebuildAllIntervals(); void rebuildAllIntervals();
QuotaPtr quota; QuotaPtr quota;

13
src/AggregateFunctions/AggregateFunctionAvg.h
View File

@ -218,9 +218,9 @@ public:
using ColVecType = ColumnVectorOrDecimal<T>; using ColVecType = ColumnVectorOrDecimal<T>;
void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const final void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const final
{ {
this->data(place).numerator += static_cast<const ColVecType &>(*columns[0]).getData()[row_num]; increment(place, static_cast<const ColVecType &>(*columns[0]).getData()[row_num]);
++this->data(place).denominator; ++this->data(place).denominator;
} }
@ -240,7 +240,7 @@ public:
sum_data.addMany(column.getData().data(), batch_size); sum_data.addMany(column.getData().data(), batch_size);
this->data(place).denominator += batch_size; this->data(place).denominator += batch_size;
} }
this->data(place).numerator += sum_data.sum; increment(place, sum_data.sum);
} }
void addBatchSinglePlaceNotNull( void addBatchSinglePlaceNotNull(
@ -270,7 +270,7 @@ public:
sum_data.addManyNotNull(column.getData().data(), null_map, batch_size); sum_data.addManyNotNull(column.getData().data(), null_map, batch_size);
this->data(place).denominator += batch_size - countBytesInFilter(null_map, batch_size); this->data(place).denominator += batch_size - countBytesInFilter(null_map, batch_size);
} }
this->data(place).numerator += sum_data.sum; increment(place, sum_data.sum);
} }
String getName() const override { return "avg"; } String getName() const override { return "avg"; }
@ -298,5 +298,10 @@ public:
#endif #endif
private:
void NO_SANITIZE_UNDEFINED increment(AggregateDataPtr __restrict place, Numerator inc) const
{
this->data(place).numerator += inc;
}
}; };
} }

11
src/CMakeLists.txt
View File

@ -106,6 +106,10 @@ if (USE_AWS_S3)
add_headers_and_sources(dbms Disks/S3) add_headers_and_sources(dbms Disks/S3)
endif() endif()
if (USE_AZURE_BLOB_STORAGE)
add_headers_and_sources(dbms Disks/BlobStorage)
endif()
if (USE_HDFS) if (USE_HDFS)
add_headers_and_sources(dbms Storages/HDFS) add_headers_and_sources(dbms Storages/HDFS)
add_headers_and_sources(dbms Disks/HDFS) add_headers_and_sources(dbms Disks/HDFS)
@ -450,6 +454,11 @@ if (USE_AWS_S3)
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_INCLUDE_DIR}) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_INCLUDE_DIR})
endif() endif()
if (USE_AZURE_BLOB_STORAGE)
target_link_libraries (clickhouse_common_io PUBLIC ${AZURE_BLOB_STORAGE_LIBRARY})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AZURE_SDK_INCLUDES})
endif()
if (USE_S2_GEOMETRY) if (USE_S2_GEOMETRY)
dbms_target_link_libraries (PUBLIC ${S2_GEOMETRY_LIBRARY}) dbms_target_link_libraries (PUBLIC ${S2_GEOMETRY_LIBRARY})
dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${S2_GEOMETRY_INCLUDE_DIR}) dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${S2_GEOMETRY_INCLUDE_DIR})
@ -517,6 +526,8 @@ if (USE_BZIP2)
target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BZIP2_INCLUDE_DIR}) target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BZIP2_INCLUDE_DIR})
endif() endif()
dbms_target_link_libraries(PUBLIC consistent-hashing)
include ("${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake") include ("${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake")
if (ENABLE_TESTS AND USE_GTEST) if (ENABLE_TESTS AND USE_GTEST)

11
src/Client/ClientBase.cpp
View File

@ -1414,9 +1414,6 @@ void ClientBase::runInteractive()
highlight_callback = highlight; highlight_callback = highlight;
ReplxxLineReader lr(*suggest, history_file, config().has("multiline"), query_extenders, query_delimiters, highlight_callback); ReplxxLineReader lr(*suggest, history_file, config().has("multiline"), query_extenders, query_delimiters, highlight_callback);
#elif defined(USE_READLINE) && USE_READLINE
ReadlineLineReader lr(*suggest, history_file, config().has("multiline"), query_extenders, query_delimiters);
#else #else
LineReader lr(history_file, config().has("multiline"), query_extenders, query_delimiters); LineReader lr(history_file, config().has("multiline"), query_extenders, query_delimiters);
#endif #endif
@ -1494,17 +1491,14 @@ void ClientBase::runNonInteractive()
{ {
auto process_multi_query_from_file = [&](const String & file) auto process_multi_query_from_file = [&](const String & file)
{ {
auto text = getQueryTextPrefix();
String queries_from_file; String queries_from_file;
ReadBufferFromFile in(file); ReadBufferFromFile in(file);
readStringUntilEOF(queries_from_file, in); readStringUntilEOF(queries_from_file, in);
text += queries_from_file; return executeMultiQuery(queries_from_file);
return executeMultiQuery(text);
}; };
/// Read all queries into `text`.
for (const auto & queries_file : queries_files) for (const auto & queries_file : queries_files)
{ {
for (const auto & interleave_file : interleave_queries_files) for (const auto & interleave_file : interleave_queries_files)
@ -1519,9 +1513,6 @@ void ClientBase::runNonInteractive()
} }
String text; String text;
if (is_multiquery)
text = getQueryTextPrefix();
if (config().has("query")) if (config().has("query"))
{ {
text += config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query. text += config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query.

6
src/Client/ClientBase.h
View File

@ -78,9 +78,6 @@ protected:
String & query_to_execute, ASTPtr & parsed_query, const String & all_queries_text, String & query_to_execute, ASTPtr & parsed_query, const String & all_queries_text,
std::optional<Exception> & current_exception); std::optional<Exception> & current_exception);
/// For non-interactive multi-query mode get queries text prefix.
virtual String getQueryTextPrefix() { return ""; }
static void clearTerminal(); static void clearTerminal();
void showClientVersion(); void showClientVersion();
@ -100,9 +97,10 @@ protected:
const std::vector<Arguments> & external_tables_arguments) = 0; const std::vector<Arguments> & external_tables_arguments) = 0;
virtual void processConfig() = 0; virtual void processConfig() = 0;
private: protected:
bool processQueryText(const String & text); bool processQueryText(const String & text);
private:
void receiveResult(ASTPtr parsed_query); void receiveResult(ASTPtr parsed_query);
bool receiveAndProcessPacket(ASTPtr parsed_query, bool cancelled); bool receiveAndProcessPacket(ASTPtr parsed_query, bool cancelled);
void receiveLogs(ASTPtr parsed_query); void receiveLogs(ASTPtr parsed_query);

19
src/Client/Connection.cpp
View File

@ -603,6 +603,14 @@ void Connection::sendReadTaskResponse(const String & response)
out->next(); out->next();
} }
void Connection::sendMergeTreeReadTaskResponse(const PartitionReadResponse & response)
{
writeVarUInt(Protocol::Client::MergeTreeReadTaskResponse, *out);
response.serialize(*out);
out->next();
}
void Connection::sendPreparedData(ReadBuffer & input, size_t size, const String & name) void Connection::sendPreparedData(ReadBuffer & input, size_t size, const String & name)
{ {
/// NOTE 'Throttler' is not used in this method (could use, but it's not important right now). /// NOTE 'Throttler' is not used in this method (could use, but it's not important right now).
@ -872,6 +880,10 @@ Packet Connection::receivePacket()
case Protocol::Server::ReadTaskRequest: case Protocol::Server::ReadTaskRequest:
return res; return res;
case Protocol::Server::MergeTreeReadTaskRequest:
res.request = receivePartitionReadRequest();
return res;
case Protocol::Server::ProfileEvents: case Protocol::Server::ProfileEvents:
res.block = receiveProfileEvents(); res.block = receiveProfileEvents();
return res; return res;
@ -1023,6 +1035,13 @@ ProfileInfo Connection::receiveProfileInfo() const
return profile_info; return profile_info;
} }
PartitionReadRequest Connection::receivePartitionReadRequest() const
{
PartitionReadRequest request;
request.deserialize(*in);
return request;
}
void Connection::throwUnexpectedPacket(UInt64 packet_type, const char * expected) const void Connection::throwUnexpectedPacket(UInt64 packet_type, const char * expected) const
{ {

5
src/Client/Connection.h
View File

@ -16,6 +16,8 @@
#include <Compression/ICompressionCodec.h> #include <Compression/ICompressionCodec.h>
#include <Storages/MergeTree/RequestResponse.h>
#include <atomic> #include <atomic>
#include <optional> #include <optional>
@ -104,6 +106,8 @@ public:
void sendData(const Block & block, const String & name/* = "" */, bool scalar/* = false */) override; void sendData(const Block & block, const String & name/* = "" */, bool scalar/* = false */) override;
void sendMergeTreeReadTaskResponse(const PartitionReadResponse & response) override;
void sendExternalTablesData(ExternalTablesData & data) override; void sendExternalTablesData(ExternalTablesData & data) override;
bool poll(size_t timeout_microseconds/* = 0 */) override; bool poll(size_t timeout_microseconds/* = 0 */) override;
@ -255,6 +259,7 @@ private:
std::vector<String> receiveMultistringMessage(UInt64 msg_type) const; std::vector<String> receiveMultistringMessage(UInt64 msg_type) const;
std::unique_ptr<Exception> receiveException() const; std::unique_ptr<Exception> receiveException() const;
Progress receiveProgress() const; Progress receiveProgress() const;
PartitionReadRequest receivePartitionReadRequest() const;
ProfileInfo receiveProfileInfo() const; ProfileInfo receiveProfileInfo() const;
void initInputBuffers(); void initInputBuffers();

6
src/Client/HedgedConnections.cpp
View File

@ -132,7 +132,7 @@ void HedgedConnections::sendQuery(
const String & query, const String & query,
const String & query_id, const String & query_id,
UInt64 stage, UInt64 stage,
const ClientInfo & client_info, ClientInfo & client_info,
bool with_pending_data) bool with_pending_data)
{ {
std::lock_guard lock(cancel_mutex); std::lock_guard lock(cancel_mutex);
@ -171,7 +171,9 @@ void HedgedConnections::sendQuery(
modified_settings.group_by_two_level_threshold_bytes = 0; modified_settings.group_by_two_level_threshold_bytes = 0;
} }
if (offset_states.size() > 1) const bool enable_sample_offset_parallel_processing = settings.max_parallel_replicas > 1 && !settings.allow_experimental_parallel_reading_from_replicas;
if (offset_states.size() > 1 && enable_sample_offset_parallel_processing)
{ {
modified_settings.parallel_replicas_count = offset_states.size(); modified_settings.parallel_replicas_count = offset_states.size();
modified_settings.parallel_replica_offset = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset; modified_settings.parallel_replica_offset = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset;

11
src/Client/HedgedConnections.h
View File

@ -86,7 +86,7 @@ public:
const String & query, const String & query,
const String & query_id, const String & query_id,
UInt64 stage, UInt64 stage,
const ClientInfo & client_info, ClientInfo & client_info,
bool with_pending_data) override; bool with_pending_data) override;
void sendReadTaskResponse(const String &) override void sendReadTaskResponse(const String &) override
@ -94,6 +94,11 @@ public:
throw Exception("sendReadTaskResponse in not supported with HedgedConnections", ErrorCodes::LOGICAL_ERROR); throw Exception("sendReadTaskResponse in not supported with HedgedConnections", ErrorCodes::LOGICAL_ERROR);
} }
void sendMergeTreeReadTaskResponse(PartitionReadResponse) override
{
throw Exception("sendMergeTreeReadTaskResponse in not supported with HedgedConnections", ErrorCodes::LOGICAL_ERROR);
}
Packet receivePacket() override; Packet receivePacket() override;
Packet receivePacketUnlocked(AsyncCallback async_callback, bool is_draining) override; Packet receivePacketUnlocked(AsyncCallback async_callback, bool is_draining) override;
@ -112,6 +117,8 @@ public:
bool hasActiveConnections() const override { return active_connection_count > 0; } bool hasActiveConnections() const override { return active_connection_count > 0; }
void setReplicaInfo(ReplicaInfo value) override { replica_info = value; }
private: private:
/// If we don't receive data from replica and there is no progress in query /// If we don't receive data from replica and there is no progress in query
/// execution for receive_data_timeout, we are trying to get new /// execution for receive_data_timeout, we are trying to get new
@ -199,6 +206,8 @@ private:
bool sent_query = false; bool sent_query = false;
bool cancelled = false; bool cancelled = false;
ReplicaInfo replica_info;
mutable std::mutex cancel_mutex; mutable std::mutex cancel_mutex;
}; };

17
src/Client/IConnections.h
View File

@ -1,6 +1,9 @@
#pragma once #pragma once
#include <compare>
#include <Client/Connection.h> #include <Client/Connection.h>
#include <Storages/MergeTree/RequestResponse.h>
namespace DB namespace DB
{ {
@ -27,10 +30,11 @@ public:
const String & query, const String & query,
const String & query_id, const String & query_id,
UInt64 stage, UInt64 stage,
const ClientInfo & client_info, ClientInfo & client_info,
bool with_pending_data) = 0; bool with_pending_data) = 0;
virtual void sendReadTaskResponse(const String &) = 0; virtual void sendReadTaskResponse(const String &) = 0;
virtual void sendMergeTreeReadTaskResponse(PartitionReadResponse response) = 0;
/// Get packet from any replica. /// Get packet from any replica.
virtual Packet receivePacket() = 0; virtual Packet receivePacket() = 0;
@ -56,6 +60,17 @@ public:
/// Get the replica addresses as a string. /// Get the replica addresses as a string.
virtual std::string dumpAddresses() const = 0; virtual std::string dumpAddresses() const = 0;
struct ReplicaInfo
{
size_t all_replicas_count{0};
size_t number_of_current_replica{0};
};
/// This is needed in max_parallel_replicas case.
/// We create a RemoteQueryExecutor for each replica
virtual void setReplicaInfo(ReplicaInfo value) = 0;
/// Returns the number of replicas. /// Returns the number of replicas.
virtual size_t size() const = 0; virtual size_t size() const = 0;

7
src/Client/IServerConnection.h
View File

@ -12,6 +12,8 @@
#include <IO/ConnectionTimeouts.h> #include <IO/ConnectionTimeouts.h>
#include <IO/Progress.h> #include <IO/Progress.h>
#include <Storages/MergeTree/RequestResponse.h>
#include <boost/noncopyable.hpp> #include <boost/noncopyable.hpp>
@ -32,10 +34,13 @@ struct Packet
Progress progress; Progress progress;
ProfileInfo profile_info; ProfileInfo profile_info;
std::vector<UUID> part_uuids; std::vector<UUID> part_uuids;
PartitionReadRequest request;
PartitionReadResponse response;
Packet() : type(Protocol::Server::Hello) {} Packet() : type(Protocol::Server::Hello) {}
}; };
/// Struct which represents data we are going to send for external table. /// Struct which represents data we are going to send for external table.
struct ExternalTableData struct ExternalTableData
{ {
@ -96,6 +101,8 @@ public:
/// Send all contents of external (temporary) tables. /// Send all contents of external (temporary) tables.
virtual void sendExternalTablesData(ExternalTablesData & data) = 0; virtual void sendExternalTablesData(ExternalTablesData & data) = 0;
virtual void sendMergeTreeReadTaskResponse(const PartitionReadResponse & response) = 0;
/// Check, if has data to read. /// Check, if has data to read.
virtual bool poll(size_t timeout_microseconds) = 0; virtual bool poll(size_t timeout_microseconds) = 0;

5
src/Client/LocalConnection.cpp
View File

@ -424,6 +424,11 @@ void LocalConnection::sendExternalTablesData(ExternalTablesData &)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented"); throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented");
} }
void LocalConnection::sendMergeTreeReadTaskResponse(const PartitionReadResponse &)
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented");
}
ServerConnectionPtr LocalConnection::createConnection(const ConnectionParameters &, ContextPtr current_context, bool send_progress) ServerConnectionPtr LocalConnection::createConnection(const ConnectionParameters &, ContextPtr current_context, bool send_progress)
{ {
return std::make_unique<LocalConnection>(current_context, send_progress); return std::make_unique<LocalConnection>(current_context, send_progress);

2
src/Client/LocalConnection.h
View File

@ -92,6 +92,8 @@ public:
void sendExternalTablesData(ExternalTablesData &) override; void sendExternalTablesData(ExternalTablesData &) override;
void sendMergeTreeReadTaskResponse(const PartitionReadResponse & response) override;
bool poll(size_t timeout_microseconds/* = 0 */) override; bool poll(size_t timeout_microseconds/* = 0 */) override;
bool hasReadPendingData() const override; bool hasReadPendingData() const override;

40
src/Client/MultiplexedConnections.cpp
View File

@ -1,9 +1,10 @@
#include <Client/MultiplexedConnections.h> #include <Client/MultiplexedConnections.h>
#include <Common/thread_local_rng.h>
#include <Core/Protocol.h>
#include <IO/ConnectionTimeouts.h> #include <IO/ConnectionTimeouts.h>
#include <IO/Operators.h> #include <IO/Operators.h>
#include <Common/thread_local_rng.h> #include <Interpreters/ClientInfo.h>
#include "Core/Protocol.h"
namespace DB namespace DB
{ {
@ -110,7 +111,7 @@ void MultiplexedConnections::sendQuery(
const String & query, const String & query,
const String & query_id, const String & query_id,
UInt64 stage, UInt64 stage,
const ClientInfo & client_info, ClientInfo & client_info,
bool with_pending_data) bool with_pending_data)
{ {
std::lock_guard lock(cancel_mutex); std::lock_guard lock(cancel_mutex);
@ -131,16 +132,29 @@ void MultiplexedConnections::sendQuery(
modified_settings.group_by_two_level_threshold = 0; modified_settings.group_by_two_level_threshold = 0;
modified_settings.group_by_two_level_threshold_bytes = 0; modified_settings.group_by_two_level_threshold_bytes = 0;
} }
if (settings.allow_experimental_parallel_reading_from_replicas)
{
client_info.collaborate_with_initiator = true;
client_info.count_participating_replicas = replica_info.all_replicas_count;
client_info.number_of_current_replica = replica_info.number_of_current_replica;
}
} }
const bool enable_sample_offset_parallel_processing = settings.max_parallel_replicas > 1 && !settings.allow_experimental_parallel_reading_from_replicas;
size_t num_replicas = replica_states.size(); size_t num_replicas = replica_states.size();
if (num_replicas > 1) if (num_replicas > 1)
{ {
/// Use multiple replicas for parallel query processing. if (enable_sample_offset_parallel_processing)
modified_settings.parallel_replicas_count = num_replicas; /// Use multiple replicas for parallel query processing.
modified_settings.parallel_replicas_count = num_replicas;
for (size_t i = 0; i < num_replicas; ++i) for (size_t i = 0; i < num_replicas; ++i)
{ {
modified_settings.parallel_replica_offset = i; if (enable_sample_offset_parallel_processing)
modified_settings.parallel_replica_offset = i;
replica_states[i].connection->sendQuery(timeouts, query, query_id, replica_states[i].connection->sendQuery(timeouts, query, query_id,
stage, &modified_settings, &client_info, with_pending_data); stage, &modified_settings, &client_info, with_pending_data);
} }
@ -179,6 +193,16 @@ void MultiplexedConnections::sendReadTaskResponse(const String & response)
current_connection->sendReadTaskResponse(response); current_connection->sendReadTaskResponse(response);
} }
void MultiplexedConnections::sendMergeTreeReadTaskResponse(PartitionReadResponse response)
{
std::lock_guard lock(cancel_mutex);
if (cancelled)
return;
current_connection->sendMergeTreeReadTaskResponse(response);
}
Packet MultiplexedConnections::receivePacket() Packet MultiplexedConnections::receivePacket()
{ {
std::lock_guard lock(cancel_mutex); std::lock_guard lock(cancel_mutex);
@ -234,6 +258,7 @@ Packet MultiplexedConnections::drain()
switch (packet.type) switch (packet.type)
{ {
case Protocol::Server::MergeTreeReadTaskRequest:
case Protocol::Server::ReadTaskRequest: case Protocol::Server::ReadTaskRequest:
case Protocol::Server::PartUUIDs: case Protocol::Server::PartUUIDs:
case Protocol::Server::Data: case Protocol::Server::Data:
@ -313,6 +338,7 @@ Packet MultiplexedConnections::receivePacketUnlocked(AsyncCallback async_callbac
switch (packet.type) switch (packet.type)
{ {
case Protocol::Server::MergeTreeReadTaskRequest:
case Protocol::Server::ReadTaskRequest: case Protocol::Server::ReadTaskRequest:
case Protocol::Server::PartUUIDs: case Protocol::Server::PartUUIDs:
case Protocol::Server::Data: case Protocol::Server::Data:

6
src/Client/MultiplexedConnections.h
View File

@ -38,10 +38,11 @@ public:
const String & query, const String & query,
const String & query_id, const String & query_id,
UInt64 stage, UInt64 stage,
const ClientInfo & client_info, ClientInfo & client_info,
bool with_pending_data) override; bool with_pending_data) override;
void sendReadTaskResponse(const String &) override; void sendReadTaskResponse(const String &) override;
void sendMergeTreeReadTaskResponse(PartitionReadResponse response) override;
Packet receivePacket() override; Packet receivePacket() override;
@ -62,6 +63,7 @@ public:
/// Without locking, because sendCancel() does not change the state of the replicas. /// Without locking, because sendCancel() does not change the state of the replicas.
bool hasActiveConnections() const override { return active_connection_count > 0; } bool hasActiveConnections() const override { return active_connection_count > 0; }
void setReplicaInfo(ReplicaInfo value) override { replica_info = value; }
private: private:
Packet receivePacketUnlocked(AsyncCallback async_callback, bool is_draining) override; Packet receivePacketUnlocked(AsyncCallback async_callback, bool is_draining) override;
@ -102,6 +104,8 @@ private:
bool sent_query = false; bool sent_query = false;
bool cancelled = false; bool cancelled = false;
ReplicaInfo replica_info;
/// A mutex for the sendCancel function to execute safely /// A mutex for the sendCancel function to execute safely
/// in separate thread. /// in separate thread.
mutable std::mutex cancel_mutex; mutable std::mutex cancel_mutex;

2
src/Common/ErrorCodes.cpp
View File

@ -470,6 +470,7 @@
M(497, ACCESS_DENIED) \ M(497, ACCESS_DENIED) \
M(498, LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED) \ M(498, LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED) \
M(499, S3_ERROR) \ M(499, S3_ERROR) \
M(500, BLOB_STORAGE_ERROR) \
M(501, CANNOT_CREATE_DATABASE) \ M(501, CANNOT_CREATE_DATABASE) \
M(502, CANNOT_SIGQUEUE) \ M(502, CANNOT_SIGQUEUE) \
M(503, AGGREGATE_FUNCTION_THROW) \ M(503, AGGREGATE_FUNCTION_THROW) \
@ -601,6 +602,7 @@
M(631, UNKNOWN_FILE_SIZE) \ M(631, UNKNOWN_FILE_SIZE) \
M(632, UNEXPECTED_DATA_AFTER_PARSED_VALUE) \ M(632, UNEXPECTED_DATA_AFTER_PARSED_VALUE) \
M(633, QUERY_IS_NOT_SUPPORTED_IN_WINDOW_VIEW) \ M(633, QUERY_IS_NOT_SUPPORTED_IN_WINDOW_VIEW) \
M(634, MONGODB_ERROR) \
\ \
M(999, KEEPER_EXCEPTION) \ M(999, KEEPER_EXCEPTION) \
M(1000, POCO_EXCEPTION) \ M(1000, POCO_EXCEPTION) \

1
src/Common/PoolBase.h
View File

@ -163,4 +163,3 @@ protected:
/** Creates a new object to put into the pool. */ /** Creates a new object to put into the pool. */
virtual ObjectPtr allocObject() = 0; virtual ObjectPtr allocObject() = 0;
}; };

10
src/Common/SparseHashMap.h
View File

@ -1,10 +0,0 @@
#pragma once
/// SparseHashMap is a wrapper for google::sparse_hash_map.
#include <sparsehash/sparse_hash_map>
template <class Key, class T, class HashFcn = std::hash<Key>,
class EqualKey = std::equal_to<Key>,
class Alloc = google::libc_allocator_with_realloc<std::pair<const Key, T>>>
using SparseHashMap = google::sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>;

1
src/Common/config.h.in
View File

@ -9,6 +9,7 @@
#cmakedefine01 USE_HDFS #cmakedefine01 USE_HDFS
#cmakedefine01 USE_INTERNAL_HDFS3_LIBRARY #cmakedefine01 USE_INTERNAL_HDFS3_LIBRARY
#cmakedefine01 USE_AWS_S3 #cmakedefine01 USE_AWS_S3
#cmakedefine01 USE_AZURE_BLOB_STORAGE
#cmakedefine01 USE_BROTLI #cmakedefine01 USE_BROTLI
#cmakedefine01 USE_UNWIND #cmakedefine01 USE_UNWIND
#cmakedefine01 USE_OPENCL #cmakedefine01 USE_OPENCL

2
src/Coordination/tests/gtest_coordination.cpp
View File

@ -1,6 +1,5 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <Common/config.h>
#include "config_core.h" #include "config_core.h"
#if USE_NURAFT #if USE_NURAFT
@ -15,7 +14,6 @@
#include <Coordination/WriteBufferFromNuraftBuffer.h> #include <Coordination/WriteBufferFromNuraftBuffer.h>
#include <Coordination/ReadBufferFromNuraftBuffer.h> #include <Coordination/ReadBufferFromNuraftBuffer.h>
#include <IO/ReadBufferFromString.h> #include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromString.h>
#include <Common/ZooKeeper/ZooKeeperCommon.h> #include <Common/ZooKeeper/ZooKeeperCommon.h>
#include <Common/ZooKeeper/ZooKeeperIO.h> #include <Common/ZooKeeper/ZooKeeperIO.h>
#include <Common/Exception.h> #include <Common/Exception.h>

68
src/Core/Protocol.h
View File

@ -64,24 +64,26 @@ namespace Protocol
{ {
enum Enum enum Enum
{ {
Hello = 0, /// Name, version, revision. Hello = 0, /// Name, version, revision.
Data = 1, /// A block of data (compressed or not). Data = 1, /// A block of data (compressed or not).
Exception = 2, /// The exception during query execution. Exception = 2, /// The exception during query execution.
Progress = 3, /// Query execution progress: rows read, bytes read. Progress = 3, /// Query execution progress: rows read, bytes read.
Pong = 4, /// Ping response Pong = 4, /// Ping response
EndOfStream = 5, /// All packets were transmitted EndOfStream = 5, /// All packets were transmitted
ProfileInfo = 6, /// Packet with profiling info. ProfileInfo = 6, /// Packet with profiling info.
Totals = 7, /// A block with totals (compressed or not). Totals = 7, /// A block with totals (compressed or not).
Extremes = 8, /// A block with minimums and maximums (compressed or not). Extremes = 8, /// A block with minimums and maximums (compressed or not).
TablesStatusResponse = 9, /// A response to TablesStatus request. TablesStatusResponse = 9, /// A response to TablesStatus request.
Log = 10, /// System logs of the query execution Log = 10, /// System logs of the query execution
TableColumns = 11, /// Columns' description for default values calculation TableColumns = 11, /// Columns' description for default values calculation
PartUUIDs = 12, /// List of unique parts ids. PartUUIDs = 12, /// List of unique parts ids.
ReadTaskRequest = 13, /// String (UUID) describes a request for which next task is needed ReadTaskRequest = 13, /// String (UUID) describes a request for which next task is needed
/// This is such an inverted logic, where server sends requests /// This is such an inverted logic, where server sends requests
/// And client returns back response /// And client returns back response
ProfileEvents = 14, /// Packet with profile events from server. ProfileEvents = 14, /// Packet with profile events from server.
MAX = ProfileEvents, MergeTreeReadTaskRequest = 15, /// Request from a MergeTree replica to a coordinator
MAX = MergeTreeReadTaskRequest,
}; };
/// NOTE: If the type of packet argument would be Enum, the comparison packet >= 0 && packet < 10 /// NOTE: If the type of packet argument would be Enum, the comparison packet >= 0 && packet < 10
@ -106,6 +108,7 @@ namespace Protocol
"PartUUIDs", "PartUUIDs",
"ReadTaskRequest", "ReadTaskRequest",
"ProfileEvents", "ProfileEvents",
"MergeTreeReadTaskRequest",
}; };
return packet <= MAX return packet <= MAX
? data[packet] ? data[packet]
@ -130,20 +133,20 @@ namespace Protocol
{ {
enum Enum enum Enum
{ {
Hello = 0, /// Name, version, revision, default DB Hello = 0, /// Name, version, revision, default DB
Query = 1, /// Query id, query settings, stage up to which the query must be executed, Query = 1, /// Query id, query settings, stage up to which the query must be executed,
/// whether the compression must be used, /// whether the compression must be used,
/// query text (without data for INSERTs). /// query text (without data for INSERTs).
Data = 2, /// A block of data (compressed or not). Data = 2, /// A block of data (compressed or not).
Cancel = 3, /// Cancel the query execution. Cancel = 3, /// Cancel the query execution.
Ping = 4, /// Check that connection to the server is alive. Ping = 4, /// Check that connection to the server is alive.
TablesStatusRequest = 5, /// Check status of tables on the server. TablesStatusRequest = 5, /// Check status of tables on the server.
KeepAlive = 6, /// Keep the connection alive KeepAlive = 6, /// Keep the connection alive
Scalar = 7, /// A block of data (compressed or not). Scalar = 7, /// A block of data (compressed or not).
IgnoredPartUUIDs = 8, /// List of unique parts ids to exclude from query processing IgnoredPartUUIDs = 8, /// List of unique parts ids to exclude from query processing
ReadTaskResponse = 9, /// TODO: ReadTaskResponse = 9, /// A filename to read from s3 (used in s3Cluster)
MergeTreeReadTaskResponse = 10, /// Coordinator's decision with a modified set of mark ranges allowed to read
MAX = ReadTaskResponse, MAX = MergeTreeReadTaskResponse,
}; };
inline const char * toString(UInt64 packet) inline const char * toString(UInt64 packet)
@ -159,6 +162,7 @@ namespace Protocol
"Scalar", "Scalar",
"IgnoredPartUUIDs", "IgnoredPartUUIDs",
"ReadTaskResponse", "ReadTaskResponse",
"MergeTreeReadTaskResponse"
}; };
return packet <= MAX return packet <= MAX
? data[packet] ? data[packet]

6
src/Core/ProtocolDefines.h
View File

@ -31,6 +31,9 @@
#define DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION 1 #define DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION 1
#define DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION 1
#define DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS 54453
/// Minimum revision supporting interserver secret. /// Minimum revision supporting interserver secret.
#define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441 #define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441
@ -48,6 +51,7 @@
/// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION, /// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION,
/// later is just a number for server version (one number instead of commit SHA) /// later is just a number for server version (one number instead of commit SHA)
/// for simplicity (sometimes it may be more convenient in some use cases). /// for simplicity (sometimes it may be more convenient in some use cases).
#define DBMS_TCP_PROTOCOL_VERSION 54452
#define DBMS_TCP_PROTOCOL_VERSION 54453
#define DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME 54449 #define DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME 54449

2
src/Core/Settings.h
View File

@ -126,6 +126,8 @@ class IColumn;
M(UInt64, parallel_replicas_count, 0, "", 0) \ M(UInt64, parallel_replicas_count, 0, "", 0) \
M(UInt64, parallel_replica_offset, 0, "", 0) \ M(UInt64, parallel_replica_offset, 0, "", 0) \
\ \
M(Bool, allow_experimental_parallel_reading_from_replicas, false, "If true, ClickHouse will send a SELECT query to all replicas of a table. It will work for any kind on MergeTree table.", 0) \
\
M(Bool, skip_unavailable_shards, false, "If true, ClickHouse silently skips unavailable shards and nodes unresolvable through DNS. Shard is marked as unavailable when none of the replicas can be reached.", 0) \ M(Bool, skip_unavailable_shards, false, "If true, ClickHouse silently skips unavailable shards and nodes unresolvable through DNS. Shard is marked as unavailable when none of the replicas can be reached.", 0) \
\ \
M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \ M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \

401
src/DataStreams/PushingToViewsBlockOutputStream.cpp
View File

@ -1,401 +0,0 @@
#include <DataStreams/ConvertingBlockInputStream.h>
#include <DataStreams/PushingToViewsBlockOutputStream.h>
#include <DataStreams/SquashingBlockInputStream.h>
#include <DataStreams/OneBlockInputStream.h>
#include <DataStreams/MaterializingBlockInputStream.h>
#include <DataStreams/copyData.h>
#include <DataTypes/NestedUtils.h>
#include <Interpreters/InterpreterSelectQuery.h>
#include <Interpreters/InterpreterInsertQuery.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTInsertQuery.h>
#include <Common/CurrentThread.h>
#include <Common/setThreadName.h>
#include <Common/ThreadPool.h>
#include <Common/checkStackSize.h>
#include <Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h>
#include <Storages/StorageValues.h>
#include <Storages/WindowView/StorageWindowView.h>
#include <Storages/LiveView/StorageLiveView.h>
#include <Storages/StorageMaterializedView.h>
#include <common/logger_useful.h>
namespace DB
{
PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
const StoragePtr & storage_,
const StorageMetadataPtr & metadata_snapshot_,
ContextPtr context_,
const ASTPtr & query_ptr_,
bool no_destination)
: WithContext(context_)
, storage(storage_)
, metadata_snapshot(metadata_snapshot_)
, log(&Poco::Logger::get("PushingToViewsBlockOutputStream"))
, query_ptr(query_ptr_)
{
checkStackSize();
/** TODO This is a very important line. At any insertion into the table one of streams should own lock.
* Although now any insertion into the table is done via PushingToViewsBlockOutputStream,
* but it's clear that here is not the best place for this functionality.
*/
addTableLock(
storage->lockForShare(getContext()->getInitialQueryId(), getContext()->getSettingsRef().lock_acquire_timeout));
/// If the "root" table deduplicates blocks, there are no need to make deduplication for children
/// Moreover, deduplication for AggregatingMergeTree children could produce false positives due to low size of inserting blocks
bool disable_deduplication_for_children = false;
if (!getContext()->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views)
disable_deduplication_for_children = !no_destination && storage->supportsDeduplication();
auto table_id = storage->getStorageID();
Dependencies dependencies = DatabaseCatalog::instance().getDependencies(table_id);
/// We need special context for materialized views insertions
if (!dependencies.empty())
{
select_context = Context::createCopy(context);
insert_context = Context::createCopy(context);
const auto & insert_settings = insert_context->getSettingsRef();
// Do not deduplicate insertions into MV if the main insertion is Ok
if (disable_deduplication_for_children)
insert_context->setSetting("insert_deduplicate", Field{false});
// Separate min_insert_block_size_rows/min_insert_block_size_bytes for children
if (insert_settings.min_insert_block_size_rows_for_materialized_views)
insert_context->setSetting("min_insert_block_size_rows", insert_settings.min_insert_block_size_rows_for_materialized_views.value);
if (insert_settings.min_insert_block_size_bytes_for_materialized_views)
insert_context->setSetting("min_insert_block_size_bytes", insert_settings.min_insert_block_size_bytes_for_materialized_views.value);
}
for (const auto & database_table : dependencies)
{
auto dependent_table = DatabaseCatalog::instance().getTable(database_table, getContext());
auto dependent_metadata_snapshot = dependent_table->getInMemoryMetadataPtr();
ASTPtr query;
BlockOutputStreamPtr out;
if (auto * materialized_view = dynamic_cast<StorageMaterializedView *>(dependent_table.get()))
{
addTableLock(
materialized_view->lockForShare(getContext()->getInitialQueryId(), getContext()->getSettingsRef().lock_acquire_timeout));
StoragePtr inner_table = materialized_view->getTargetTable();
auto inner_table_id = inner_table->getStorageID();
auto inner_metadata_snapshot = inner_table->getInMemoryMetadataPtr();
query = dependent_metadata_snapshot->getSelectQuery().inner_query;
std::unique_ptr<ASTInsertQuery> insert = std::make_unique<ASTInsertQuery>();
insert->table_id = inner_table_id;
/// Get list of columns we get from select query.
auto header = InterpreterSelectQuery(query, select_context, SelectQueryOptions().analyze())
.getSampleBlock();
/// Insert only columns returned by select.
auto list = std::make_shared<ASTExpressionList>();
const auto & inner_table_columns = inner_metadata_snapshot->getColumns();
for (const auto & column : header)
{
/// But skip columns which storage doesn't have.
if (inner_table_columns.hasPhysical(column.name))
list->children.emplace_back(std::make_shared<ASTIdentifier>(column.name));
}
insert->columns = std::move(list);
ASTPtr insert_query_ptr(insert.release());
InterpreterInsertQuery interpreter(insert_query_ptr, insert_context);
BlockIO io = interpreter.execute();
out = io.out;
}
else if (
dynamic_cast<const StorageLiveView *>(dependent_table.get()) || dynamic_cast<const StorageWindowView *>(dependent_table.get()))
out = std::make_shared<PushingToViewsBlockOutputStream>(
dependent_table, dependent_metadata_snapshot, insert_context, ASTPtr(), true);
else
out = std::make_shared<PushingToViewsBlockOutputStream>(
dependent_table, dependent_metadata_snapshot, insert_context, ASTPtr());
views.emplace_back(ViewInfo{std::move(query), database_table, std::move(out), nullptr, 0 /* elapsed_ms */});
}
/// Do not push to destination table if the flag is set
if (!no_destination)
{
output = storage->write(query_ptr, storage->getInMemoryMetadataPtr(), getContext());
replicated_output = dynamic_cast<ReplicatedMergeTreeBlockOutputStream *>(output.get());
}
}
Block PushingToViewsBlockOutputStream::getHeader() const
{
/// If we don't write directly to the destination
/// then expect that we're inserting with precalculated virtual columns
if (output)
return metadata_snapshot->getSampleBlock();
else
return metadata_snapshot->getSampleBlockWithVirtuals(storage->getVirtuals());
}
void PushingToViewsBlockOutputStream::write(const Block & block)
{
/** Throw an exception if the sizes of arrays - elements of nested data structures doesn't match.
* We have to make this assertion before writing to table, because storage engine may assume that they have equal sizes.
* NOTE It'd better to do this check in serialization of nested structures (in place when this assumption is required),
* but currently we don't have methods for serialization of nested structures "as a whole".
*/
Nested::validateArraySizes(block);
if (auto * live_view = dynamic_cast<StorageLiveView *>(storage.get()))
{
StorageLiveView::writeIntoLiveView(*live_view, block, getContext());
}
else if (auto * window_view = dynamic_cast<StorageWindowView *>(storage.get()))
{
StorageWindowView::writeIntoWindowView(*window_view, block, getContext());
}
else
{
if (output)
/// TODO: to support virtual and alias columns inside MVs, we should return here the inserted block extended
/// with additional columns directly from storage and pass it to MVs instead of raw block.
output->write(block);
}
/// Don't process materialized views if this block is duplicate
if (!getContext()->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views && replicated_output && replicated_output->lastBlockIsDuplicate())
return;
// Insert data into materialized views only after successful insert into main table
const Settings & settings = getContext()->getSettingsRef();
if (settings.parallel_view_processing && views.size() > 1)
{
// Push to views concurrently if enabled and more than one view is attached
ThreadPool pool(std::min(size_t(settings.max_threads), views.size()));
for (auto & view : views)
{
auto thread_group = CurrentThread::getGroup();
pool.scheduleOrThrowOnError([=, &view, this]
{
setThreadName("PushingToViews");
if (thread_group)
CurrentThread::attachToIfDetached(thread_group);
process(block, view);
});
}
// Wait for concurrent view processing
pool.wait();
}
else
{
// Process sequentially
for (auto & view : views)
{
process(block, view);
if (view.exception)
std::rethrow_exception(view.exception);
}
}
}
void PushingToViewsBlockOutputStream::writePrefix()
{
if (output)
output->writePrefix();
for (auto & view : views)
{
try
{
view.out->writePrefix();
}
catch (Exception & ex)
{
ex.addMessage("while write prefix to view " + view.table_id.getNameForLogs());
throw;
}
}
}
void PushingToViewsBlockOutputStream::writeSuffix()
{
if (output)
output->writeSuffix();
std::exception_ptr first_exception;
const Settings & settings = getContext()->getSettingsRef();
bool parallel_processing = false;
/// Run writeSuffix() for views in separate thread pool.
/// In could have been done in PushingToViewsBlockOutputStream::process, however
/// it is not good if insert into main table fail but into view succeed.
if (settings.parallel_view_processing && views.size() > 1)
{
parallel_processing = true;
// Push to views concurrently if enabled and more than one view is attached
ThreadPool pool(std::min(size_t(settings.max_threads), views.size()));
auto thread_group = CurrentThread::getGroup();
for (auto & view : views)
{
if (view.exception)
continue;
pool.scheduleOrThrowOnError([thread_group, &view, this]
{
setThreadName("PushingToViews");
if (thread_group)
CurrentThread::attachToIfDetached(thread_group);
Stopwatch watch;
try
{
view.out->writeSuffix();
}
catch (...)
{
view.exception = std::current_exception();
}
view.elapsed_ms += watch.elapsedMilliseconds();
LOG_TRACE(log, "Pushing from {} to {} took {} ms.",
storage->getStorageID().getNameForLogs(),
view.table_id.getNameForLogs(),
view.elapsed_ms);
});
}
// Wait for concurrent view processing
pool.wait();
}
for (auto & view : views)
{
if (view.exception)
{
if (!first_exception)
first_exception = view.exception;
continue;
}
if (parallel_processing)
continue;
Stopwatch watch;
try
{
view.out->writeSuffix();
}
catch (Exception & ex)
{
ex.addMessage("while write prefix to view " + view.table_id.getNameForLogs());
throw;
}
view.elapsed_ms += watch.elapsedMilliseconds();
LOG_TRACE(log, "Pushing from {} to {} took {} ms.",
storage->getStorageID().getNameForLogs(),
view.table_id.getNameForLogs(),
view.elapsed_ms);
}
if (first_exception)
std::rethrow_exception(first_exception);
UInt64 milliseconds = main_watch.elapsedMilliseconds();
if (views.size() > 1)
{
LOG_DEBUG(log, "Pushing from {} to {} views took {} ms.",
storage->getStorageID().getNameForLogs(), views.size(),
milliseconds);
}
}
void PushingToViewsBlockOutputStream::flush()
{
if (output)
output->flush();
for (auto & view : views)
view.out->flush();
}
void PushingToViewsBlockOutputStream::process(const Block & block, ViewInfo & view)
{
Stopwatch watch;
try
{
BlockInputStreamPtr in;
/// We need keep InterpreterSelectQuery, until the processing will be finished, since:
///
/// - We copy Context inside InterpreterSelectQuery to support
/// modification of context (Settings) for subqueries
/// - InterpreterSelectQuery lives shorter than query pipeline.
/// It's used just to build the query pipeline and no longer needed
/// - ExpressionAnalyzer and then, Functions, that created in InterpreterSelectQuery,
/// **can** take a reference to Context from InterpreterSelectQuery
/// (the problem raises only when function uses context from the
/// execute*() method, like FunctionDictGet do)
/// - These objects live inside query pipeline (DataStreams) and the reference become dangling.
std::optional<InterpreterSelectQuery> select;
if (view.query)
{
/// We create a table with the same name as original table and the same alias columns,
/// but it will contain single block (that is INSERT-ed into main table).
/// InterpreterSelectQuery will do processing of alias columns.
auto local_context = Context::createCopy(select_context);
local_context->addViewSource(
StorageValues::create(storage->getStorageID(), metadata_snapshot->getColumns(), block, storage->getVirtuals()));
select.emplace(view.query, local_context, SelectQueryOptions());
in = std::make_shared<MaterializingBlockInputStream>(select->execute().getInputStream());
/// Squashing is needed here because the materialized view query can generate a lot of blocks
/// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
/// and two-level aggregation is triggered).
in = std::make_shared<SquashingBlockInputStream>(
in, getContext()->getSettingsRef().min_insert_block_size_rows, getContext()->getSettingsRef().min_insert_block_size_bytes);
in = std::make_shared<ConvertingBlockInputStream>(in, view.out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Name);
}
else
in = std::make_shared<OneBlockInputStream>(block);
in->readPrefix();
while (Block result_block = in->read())
{
Nested::validateArraySizes(result_block);
view.out->write(result_block);
}
in->readSuffix();
}
catch (Exception & ex)
{
ex.addMessage("while pushing to view " + view.table_id.getNameForLogs());
view.exception = std::current_exception();
}
catch (...)
{
view.exception = std::current_exception();
}
view.elapsed_ms += watch.elapsedMilliseconds();
}
}

10
src/DataTypes/NestedUtils.cpp
View File

@ -45,6 +45,15 @@ std::pair<std::string, std::string> splitName(const std::string & name)
return {name.substr(0, idx), name.substr(idx + 1)}; return {name.substr(0, idx), name.substr(idx + 1)};
} }
std::pair<std::string_view, std::string_view> splitName(const std::string_view & name)
{
auto idx = name.find_first_of('.');
if (idx == std::string::npos || idx == 0 || idx + 1 == name.size())
return {name, {}};
return {name.substr(0, idx), name.substr(idx + 1)};
}
std::string extractTableName(const std::string & nested_name) std::string extractTableName(const std::string & nested_name)
{ {
@ -211,6 +220,7 @@ void validateArraySizes(const Block & block)
} }
} }
std::unordered_set<String> getAllTableNames(const Block & block) std::unordered_set<String> getAllTableNames(const Block & block)
{ {
std::unordered_set<String> nested_table_names; std::unordered_set<String> nested_table_names;

1
src/DataTypes/NestedUtils.h
View File

@ -12,6 +12,7 @@ namespace Nested
std::string concatenateName(const std::string & nested_table_name, const std::string & nested_field_name); std::string concatenateName(const std::string & nested_table_name, const std::string & nested_field_name);
std::pair<std::string, std::string> splitName(const std::string & name); std::pair<std::string, std::string> splitName(const std::string & name);
std::pair<std::string_view, std::string_view> splitName(const std::string_view & name);
/// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot. /// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot.
std::string extractTableName(const std::string & nested_name); std::string extractTableName(const std::string & nested_name);

1
src/Databases/DatabaseAtomic.cpp
View File

@ -43,6 +43,7 @@ DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, c
, db_uuid(uuid) , db_uuid(uuid)
{ {
assert(db_uuid != UUIDHelpers::Nil); assert(db_uuid != UUIDHelpers::Nil);
fs::create_directories(fs::path(getContext()->getPath()) / "metadata");
fs::create_directories(path_to_table_symlinks); fs::create_directories(path_to_table_symlinks);
tryCreateMetadataSymlink(); tryCreateMetadataSymlink();
} }

4
src/Databases/DatabaseFactory.cpp
View File

@ -117,6 +117,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String
static const std::unordered_set<std::string_view> engines_with_arguments{"MySQL", "MaterializeMySQL", "MaterializedMySQL", static const std::unordered_set<std::string_view> engines_with_arguments{"MySQL", "MaterializeMySQL", "MaterializedMySQL",
"Lazy", "Replicated", "PostgreSQL", "MaterializedPostgreSQL", "SQLite"}; "Lazy", "Replicated", "PostgreSQL", "MaterializedPostgreSQL", "SQLite"};
static const std::unordered_set<std::string_view> engines_with_table_overrides{"MaterializeMySQL", "MaterializedMySQL"};
bool engine_may_have_arguments = engines_with_arguments.contains(engine_name); bool engine_may_have_arguments = engines_with_arguments.contains(engine_name);
if (engine_define->engine->arguments && !engine_may_have_arguments) if (engine_define->engine->arguments && !engine_may_have_arguments)
@ -131,6 +132,9 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String
throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_AST, throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_AST,
"Database engine `{}` cannot have parameters, primary_key, order_by, sample_by, settings", engine_name); "Database engine `{}` cannot have parameters, primary_key, order_by, sample_by, settings", engine_name);
if (create.table_overrides && !engines_with_table_overrides.contains(engine_name))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database engine `{}` cannot have table overrides", engine_name);
if (engine_name == "Ordinary") if (engine_name == "Ordinary")
return std::make_shared<DatabaseOrdinary>(database_name, metadata_path, context); return std::make_shared<DatabaseOrdinary>(database_name, metadata_path, context);
else if (engine_name == "Atomic") else if (engine_name == "Atomic")

1
src/Databases/MySQL/DatabaseMaterializedMySQL.cpp
View File

@ -9,7 +9,6 @@
# include <Databases/MySQL/MaterializedMySQLSyncThread.h> # include <Databases/MySQL/MaterializedMySQLSyncThread.h>
# include <Parsers/ASTCreateQuery.h> # include <Parsers/ASTCreateQuery.h>
# include <Storages/StorageMaterializedMySQL.h> # include <Storages/StorageMaterializedMySQL.h>
# include <Poco/Logger.h>
# include <Common/setThreadName.h> # include <Common/setThreadName.h>
# include <filesystem> # include <filesystem>

2
src/Dictionaries/HashedArrayDictionary.h
View File

@ -5,8 +5,6 @@
#include <variant> #include <variant>
#include <optional> #include <optional>
#include <Common/SparseHashMap.h>
#include <Common/HashTable/HashMap.h> #include <Common/HashTable/HashMap.h>
#include <Common/HashTable/HashSet.h> #include <Common/HashTable/HashSet.h>
#include <Core/Block.h> #include <Core/Block.h>

18
src/Dictionaries/HashedDictionary.h
View File

@ -4,8 +4,7 @@
#include <memory> #include <memory>
#include <variant> #include <variant>
#include <optional> #include <optional>
#include <sparsehash/sparse_hash_map>
#include <Common/SparseHashMap.h>
#include <Common/HashTable/HashMap.h> #include <Common/HashTable/HashMap.h>
#include <Common/HashTable/HashSet.h> #include <Common/HashTable/HashSet.h>
@ -124,11 +123,22 @@ private:
HashMap<UInt64, Value>, HashMap<UInt64, Value>,
HashMapWithSavedHash<StringRef, Value, DefaultHash<StringRef>>>; HashMapWithSavedHash<StringRef, Value, DefaultHash<StringRef>>>;
/// Here we use sparse_hash_map with DefaultHash<> for the following reasons:
///
/// - DefaultHash<> is used for HashMap
/// - DefaultHash<> (from HashTable/Hash.h> works better then std::hash<>
/// in case of sequential set of keys, but with random access to this set, i.e.
///
/// SELECT number FROM numbers(3000000) ORDER BY rand()
///
/// And even though std::hash<> works better in some other cases,
/// DefaultHash<> is preferred since the difference for this particular
/// case is significant, i.e. it can be 10x+.
template <typename Value> template <typename Value>
using CollectionTypeSparse = std::conditional_t< using CollectionTypeSparse = std::conditional_t<
dictionary_key_type == DictionaryKeyType::Simple, dictionary_key_type == DictionaryKeyType::Simple,
SparseHashMap<UInt64, Value>, google::sparse_hash_map<UInt64, Value, DefaultHash<KeyType>>,
SparseHashMap<StringRef, Value>>; google::sparse_hash_map<StringRef, Value, DefaultHash<KeyType>>>;
template <typename Value> template <typename Value>
using CollectionType = std::conditional_t<sparse, CollectionTypeSparse<Value>, CollectionTypeNonSparse<Value>>; using CollectionType = std::conditional_t<sparse, CollectionTypeSparse<Value>, CollectionTypeNonSparse<Value>>;

145
src/Disks/BlobStorage/BlobStorageAuth.cpp Normal file
View File

@ -0,0 +1,145 @@
#include <Disks/BlobStorage/BlobStorageAuth.h>
#if USE_AZURE_BLOB_STORAGE
#include <optional>
#include <re2/re2.h>
#include <azure/identity/managed_identity_credential.hpp>
using namespace Azure::Storage::Blobs;
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
struct BlobStorageEndpoint
{
const String storage_account_url;
const String container_name;
const std::optional<bool> container_already_exists;
};
void validateStorageAccountUrl(const String & storage_account_url)
{
const auto * storage_account_url_pattern_str = R"(http(()|s)://[a-z0-9-.:]+(()|/)[a-z0-9]*(()|/))";
static const RE2 storage_account_url_pattern(storage_account_url_pattern_str);
if (!re2::RE2::FullMatch(storage_account_url, storage_account_url_pattern))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Blob Storage URL is not valid, should follow the format: {}, got: {}", storage_account_url_pattern_str, storage_account_url);
}
void validateContainerName(const String & container_name)
{
auto len = container_name.length();
if (len < 3 || len > 64)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Blob Storage container name is not valid, should have length between 3 and 64, but has length: {}", len);
const auto * container_name_pattern_str = R"([a-z][a-z0-9-]+)";
static const RE2 container_name_pattern(container_name_pattern_str);
if (!re2::RE2::FullMatch(container_name, container_name_pattern))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Blob Storage container name is not valid, should follow the format: {}, got: {}", container_name_pattern_str, container_name);
}
BlobStorageEndpoint processBlobStorageEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
String storage_account_url = config.getString(config_prefix + ".storage_account_url");
validateStorageAccountUrl(storage_account_url);
String container_name = config.getString(config_prefix + ".container_name", "default-container");
validateContainerName(container_name);
std::optional<bool> container_already_exists {};
if (config.has(config_prefix + ".container_already_exists"))
container_already_exists = {config.getBool(config_prefix + ".container_already_exists")};
return {storage_account_url, container_name, container_already_exists};
}
template <class T>
std::shared_ptr<T> getClientWithConnectionString(const String & connection_str, const String & container_name) = delete;
template<>
std::shared_ptr<BlobServiceClient> getClientWithConnectionString(
const String & connection_str, const String & /*container_name*/)
{
return std::make_shared<BlobServiceClient>(BlobServiceClient::CreateFromConnectionString(connection_str));
}
template<>
std::shared_ptr<BlobContainerClient> getClientWithConnectionString(
const String & connection_str, const String & container_name)
{
return std::make_shared<BlobContainerClient>(BlobContainerClient::CreateFromConnectionString(connection_str, container_name));
}
template <class T>
std::shared_ptr<T> getBlobStorageClientWithAuth(
const String & url, const String & container_name, const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
if (config.has(config_prefix + ".connection_string"))
{
String connection_str = config.getString(config_prefix + ".connection_string");
return getClientWithConnectionString<T>(connection_str, container_name);
}
if (config.has(config_prefix + ".account_key") && config.has(config_prefix + ".account_name"))
{
auto storage_shared_key_credential = std::make_shared<Azure::Storage::StorageSharedKeyCredential>(
config.getString(config_prefix + ".account_name"),
config.getString(config_prefix + ".account_key")
);
return std::make_shared<T>(url, storage_shared_key_credential);
}
auto managed_identity_credential = std::make_shared<Azure::Identity::ManagedIdentityCredential>();
return std::make_shared<T>(url, managed_identity_credential);
}
std::shared_ptr<BlobContainerClient> getBlobContainerClient(
const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
auto endpoint = processBlobStorageEndpoint(config, config_prefix);
auto container_name = endpoint.container_name;
auto final_url = endpoint.storage_account_url
+ (endpoint.storage_account_url.back() == '/' ? "" : "/")
+ container_name;
if (endpoint.container_already_exists.value_or(false))
return getBlobStorageClientWithAuth<BlobContainerClient>(final_url, container_name, config, config_prefix);
auto blob_service_client = getBlobStorageClientWithAuth<BlobServiceClient>(endpoint.storage_account_url, container_name, config, config_prefix);
if (!endpoint.container_already_exists.has_value())
{
ListBlobContainersOptions blob_containers_list_options;
blob_containers_list_options.Prefix = container_name;
blob_containers_list_options.PageSizeHint = 1;
auto blob_containers = blob_service_client->ListBlobContainers().BlobContainers;
for (const auto & blob_container : blob_containers)
{
if (blob_container.Name == endpoint.container_name)
return getBlobStorageClientWithAuth<BlobContainerClient>(final_url, container_name, config, config_prefix);
}
}
return std::make_shared<BlobContainerClient>(
blob_service_client->CreateBlobContainer(container_name).Value);
}
}
#endif

Some files were not shown because too many files have changed in this diff Show More