Merge branch 'master' into use-dag-in-key-condition

This commit is contained in:
Nikolai Kochetov 2022-07-25 14:30:22 +02:00 committed by GitHub
commit b70be40804
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
303 changed files with 5115 additions and 1287 deletions

View File

@ -394,7 +394,7 @@ jobs:
- name: Set envs - name: Set envs
run: | run: |
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
CHECK_NAME=ClickHouse build check (actions) CHECK_NAME=ClickHouse build check
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
TEMP_PATH=${{runner.temp}}/report_check TEMP_PATH=${{runner.temp}}/report_check
NEEDS_DATA_PATH=${{runner.temp}}/needs.json NEEDS_DATA_PATH=${{runner.temp}}/needs.json
@ -437,7 +437,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (address, actions) CHECK_NAME=Stateless tests (address)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -477,7 +477,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_debug TEMP_PATH=${{runner.temp}}/stateful_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (debug, actions) CHECK_NAME=Stateful tests (debug)
REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -521,7 +521,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_thread TEMP_PATH=${{runner.temp}}/stress_thread
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (thread, actions) CHECK_NAME=Stress test (thread)
REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -560,7 +560,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_release TEMP_PATH=${{runner.temp}}/integration_tests_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (release, actions) CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports

View File

@ -102,6 +102,9 @@ jobs:
run: | run: |
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{ runner.temp }}/style_check TEMP_PATH=${{ runner.temp }}/style_check
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
RCSK
EOF EOF
- name: Download changed images - name: Download changed images
# even if artifact does not exist, e.g. on `do not test` label or failed Docker job # even if artifact does not exist, e.g. on `do not test` label or failed Docker job

View File

@ -108,7 +108,7 @@ jobs:
- name: Style Check - name: Style Check
run: | run: |
cd "$GITHUB_WORKSPACE/tests/ci" cd "$GITHUB_WORKSPACE/tests/ci"
python3 style_check.py python3 style_check.py --no-push
- name: Cleanup - name: Cleanup
if: always() if: always()
run: | run: |
@ -971,7 +971,7 @@ jobs:
- name: Set envs - name: Set envs
run: | run: |
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
CHECK_NAME=ClickHouse build check (actions) CHECK_NAME=ClickHouse build check
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
TEMP_PATH=${{runner.temp}}/report_check TEMP_PATH=${{runner.temp}}/report_check
@ -1020,7 +1020,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/report_check TEMP_PATH=${{runner.temp}}/report_check
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=ClickHouse special build check (actions) CHECK_NAME=ClickHouse special build check
NEEDS_DATA_PATH=${{runner.temp}}/needs.json NEEDS_DATA_PATH=${{runner.temp}}/needs.json
EOF EOF
- name: Download json reports - name: Download json reports
@ -1061,7 +1061,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_release TEMP_PATH=${{runner.temp}}/stateless_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, actions) CHECK_NAME=Stateless tests (release)
REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -1098,7 +1098,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_release_database_ordinary TEMP_PATH=${{runner.temp}}/stateless_release_database_ordinary
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, DatabaseOrdinary, actions) CHECK_NAME=Stateless tests (release, DatabaseOrdinary)
REPO_COPY=${{runner.temp}}/stateless_release_database_ordinary/ClickHouse REPO_COPY=${{runner.temp}}/stateless_release_database_ordinary/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -1135,7 +1135,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_s3_storage TEMP_PATH=${{runner.temp}}/stateless_s3_storage
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, s3 storage, actions) CHECK_NAME=Stateless tests (release, s3 storage)
REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -1172,7 +1172,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_release TEMP_PATH=${{runner.temp}}/stateless_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (aarch64, actions) CHECK_NAME=Stateless tests (aarch64)
REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -1209,7 +1209,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (address, actions) CHECK_NAME=Stateless tests (address)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -1248,7 +1248,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (address, actions) CHECK_NAME=Stateless tests (address)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -1287,7 +1287,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (thread, actions) CHECK_NAME=Stateless tests (thread)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -1326,7 +1326,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (thread, actions) CHECK_NAME=Stateless tests (thread)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -1365,7 +1365,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (thread, actions) CHECK_NAME=Stateless tests (thread)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
@ -1404,7 +1404,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_ubsan TEMP_PATH=${{runner.temp}}/stateless_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (ubsan, actions) CHECK_NAME=Stateless tests (ubsan)
REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -1441,7 +1441,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (memory, actions) CHECK_NAME=Stateless tests (memory)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -1480,7 +1480,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (memory, actions) CHECK_NAME=Stateless tests (memory)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -1519,7 +1519,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (memory, actions) CHECK_NAME=Stateless tests (memory)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
@ -1558,7 +1558,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug, actions) CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -1597,7 +1597,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug, actions) CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -1636,7 +1636,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug, actions) CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
@ -1678,7 +1678,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_release TEMP_PATH=${{runner.temp}}/stateful_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (release, actions) CHECK_NAME=Stateful tests (release)
REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1715,7 +1715,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_release_database_ordinary TEMP_PATH=${{runner.temp}}/stateful_release_database_ordinary
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (release, DatabaseOrdinary, actions) CHECK_NAME=Stateful tests (release, DatabaseOrdinary)
REPO_COPY=${{runner.temp}}/stateful_release_database_ordinary/ClickHouse REPO_COPY=${{runner.temp}}/stateful_release_database_ordinary/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1752,7 +1752,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_release TEMP_PATH=${{runner.temp}}/stateful_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (aarch64, actions) CHECK_NAME=Stateful tests (aarch64)
REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1789,7 +1789,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_debug TEMP_PATH=${{runner.temp}}/stateful_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (address, actions) CHECK_NAME=Stateful tests (address)
REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1826,7 +1826,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_tsan TEMP_PATH=${{runner.temp}}/stateful_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (thread, actions) CHECK_NAME=Stateful tests (thread)
REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1863,7 +1863,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_msan TEMP_PATH=${{runner.temp}}/stateful_msan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (memory, actions) CHECK_NAME=Stateful tests (memory)
REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1900,7 +1900,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_ubsan TEMP_PATH=${{runner.temp}}/stateful_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (ubsan, actions) CHECK_NAME=Stateful tests (ubsan)
REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1937,7 +1937,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_debug TEMP_PATH=${{runner.temp}}/stateful_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (debug, actions) CHECK_NAME=Stateful tests (debug)
REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1977,7 +1977,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_thread TEMP_PATH=${{runner.temp}}/stress_thread
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (address, actions) CHECK_NAME=Stress test (address)
REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2017,7 +2017,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_thread TEMP_PATH=${{runner.temp}}/stress_thread
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (thread, actions) CHECK_NAME=Stress test (thread)
REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2053,7 +2053,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_memory TEMP_PATH=${{runner.temp}}/stress_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (memory, actions) CHECK_NAME=Stress test (memory)
REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2089,7 +2089,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_undefined TEMP_PATH=${{runner.temp}}/stress_undefined
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (undefined, actions) CHECK_NAME=Stress test (undefined)
REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2125,7 +2125,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_debug TEMP_PATH=${{runner.temp}}/stress_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (debug, actions) CHECK_NAME=Stress test (debug)
REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2164,7 +2164,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, actions) CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=3 RUN_BY_HASH_TOTAL=3
@ -2202,7 +2202,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, actions) CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=3 RUN_BY_HASH_TOTAL=3
@ -2240,7 +2240,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, actions) CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=3 RUN_BY_HASH_TOTAL=3
@ -2278,7 +2278,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -2316,7 +2316,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -2354,7 +2354,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -2392,7 +2392,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=3 RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -2430,7 +2430,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_release TEMP_PATH=${{runner.temp}}/integration_tests_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (release, actions) CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=2 RUN_BY_HASH_TOTAL=2
@ -2468,7 +2468,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_release TEMP_PATH=${{runner.temp}}/integration_tests_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (release, actions) CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=2 RUN_BY_HASH_TOTAL=2
@ -2509,7 +2509,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=AST fuzzer (ASan, actions) CHECK_NAME=AST fuzzer (ASan)
REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2545,7 +2545,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=AST fuzzer (TSan, actions) CHECK_NAME=AST fuzzer (TSan)
REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2581,7 +2581,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=AST fuzzer (UBSan, actions) CHECK_NAME=AST fuzzer (UBSan)
REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2617,7 +2617,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=AST fuzzer (MSan, actions) CHECK_NAME=AST fuzzer (MSan)
REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2653,7 +2653,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=AST fuzzer (debug, actions) CHECK_NAME=AST fuzzer (debug)
REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2692,7 +2692,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/unit_tests_asan TEMP_PATH=${{runner.temp}}/unit_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Unit tests (asan, actions) CHECK_NAME=Unit tests (asan)
REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2728,7 +2728,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/unit_tests_asan TEMP_PATH=${{runner.temp}}/unit_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Unit tests (release-clang, actions) CHECK_NAME=Unit tests (release-clang)
REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2764,7 +2764,7 @@ jobs:
# cat >> "$GITHUB_ENV" << 'EOF' # cat >> "$GITHUB_ENV" << 'EOF'
# TEMP_PATH=${{runner.temp}}/unit_tests_asan # TEMP_PATH=${{runner.temp}}/unit_tests_asan
# REPORTS_PATH=${{runner.temp}}/reports_dir # REPORTS_PATH=${{runner.temp}}/reports_dir
# CHECK_NAME=Unit tests (release-gcc, actions) # CHECK_NAME=Unit tests (release-gcc)
# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse # REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse
# EOF # EOF
# - name: Download json reports # - name: Download json reports
@ -2800,7 +2800,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/unit_tests_tsan TEMP_PATH=${{runner.temp}}/unit_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Unit tests (tsan, actions) CHECK_NAME=Unit tests (tsan)
REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2836,7 +2836,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/unit_tests_msan TEMP_PATH=${{runner.temp}}/unit_tests_msan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Unit tests (msan, actions) CHECK_NAME=Unit tests (msan)
REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2872,7 +2872,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/unit_tests_ubsan TEMP_PATH=${{runner.temp}}/unit_tests_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Unit tests (ubsan, actions) CHECK_NAME=Unit tests (ubsan)
REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports

View File

@ -118,6 +118,9 @@ jobs:
run: | run: |
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{ runner.temp }}/style_check TEMP_PATH=${{ runner.temp }}/style_check
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
RCSK
EOF EOF
- name: Download changed images - name: Download changed images
# even if artifact does not exist, e.g. on `do not test` label or failed Docker job # even if artifact does not exist, e.g. on `do not test` label or failed Docker job
@ -1026,7 +1029,7 @@ jobs:
- name: Set envs - name: Set envs
run: | run: |
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
CHECK_NAME=ClickHouse build check (actions) CHECK_NAME=ClickHouse build check
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
TEMP_PATH=${{runner.temp}}/report_check TEMP_PATH=${{runner.temp}}/report_check
NEEDS_DATA_PATH=${{runner.temp}}/needs.json NEEDS_DATA_PATH=${{runner.temp}}/needs.json
@ -1075,7 +1078,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/report_check TEMP_PATH=${{runner.temp}}/report_check
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=ClickHouse special build check (actions) CHECK_NAME=ClickHouse special build check
NEEDS_DATA_PATH=${{runner.temp}}/needs.json NEEDS_DATA_PATH=${{runner.temp}}/needs.json
EOF EOF
- name: Download json reports - name: Download json reports
@ -1116,7 +1119,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_release TEMP_PATH=${{runner.temp}}/stateless_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, actions) CHECK_NAME=Stateless tests (release)
REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -1153,7 +1156,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_database_replicated TEMP_PATH=${{runner.temp}}/stateless_database_replicated
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, DatabaseReplicated, actions) CHECK_NAME=Stateless tests (release, DatabaseReplicated)
REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -1192,7 +1195,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_database_replicated TEMP_PATH=${{runner.temp}}/stateless_database_replicated
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, DatabaseReplicated, actions) CHECK_NAME=Stateless tests (release, DatabaseReplicated)
REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -1231,7 +1234,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_wide_parts TEMP_PATH=${{runner.temp}}/stateless_wide_parts
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, wide parts enabled, actions) CHECK_NAME=Stateless tests (release, wide parts enabled)
REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -1268,7 +1271,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_s3_storage TEMP_PATH=${{runner.temp}}/stateless_s3_storage
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, s3 storage, actions) CHECK_NAME=Stateless tests (release, s3 storage)
REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -1305,7 +1308,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_release TEMP_PATH=${{runner.temp}}/stateless_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (aarch64, actions) CHECK_NAME=Stateless tests (aarch64)
REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -1342,7 +1345,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (address, actions) CHECK_NAME=Stateless tests (address)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -1381,7 +1384,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (address, actions) CHECK_NAME=Stateless tests (address)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -1420,7 +1423,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (thread, actions) CHECK_NAME=Stateless tests (thread)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -1459,7 +1462,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (thread, actions) CHECK_NAME=Stateless tests (thread)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -1498,7 +1501,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (thread, actions) CHECK_NAME=Stateless tests (thread)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
@ -1537,7 +1540,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_ubsan TEMP_PATH=${{runner.temp}}/stateless_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (ubsan, actions) CHECK_NAME=Stateless tests (ubsan)
REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -1574,7 +1577,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (memory, actions) CHECK_NAME=Stateless tests (memory)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -1613,7 +1616,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (memory, actions) CHECK_NAME=Stateless tests (memory)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -1652,7 +1655,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (memory, actions) CHECK_NAME=Stateless tests (memory)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
@ -1691,7 +1694,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug, actions) CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -1730,7 +1733,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug, actions) CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -1769,7 +1772,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug, actions) CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
@ -1808,7 +1811,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_flaky_asan TEMP_PATH=${{runner.temp}}/stateless_flaky_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests flaky check (address, actions) CHECK_NAME=Stateless tests flaky check (address)
REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1844,7 +1847,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/tests_bugfix_check TEMP_PATH=${{runner.temp}}/tests_bugfix_check
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Tests bugfix validate check (actions) CHECK_NAME=tests bugfix validate check
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse
EOF EOF
@ -1866,12 +1869,12 @@ jobs:
TEMP_PATH="${TEMP_PATH}/integration" \ TEMP_PATH="${TEMP_PATH}/integration" \
REPORTS_PATH="${REPORTS_PATH}/integration" \ REPORTS_PATH="${REPORTS_PATH}/integration" \
python3 integration_test_check.py "Integration tests bugfix validate check" \ python3 integration_test_check.py "Integration $CHECK_NAME" \
--validate-bugfix --post-commit-status=file || echo 'ignore exit code' --validate-bugfix --post-commit-status=file || echo 'ignore exit code'
TEMP_PATH="${TEMP_PATH}/stateless" \ TEMP_PATH="${TEMP_PATH}/stateless" \
REPORTS_PATH="${REPORTS_PATH}/stateless" \ REPORTS_PATH="${REPORTS_PATH}/stateless" \
python3 functional_test_check.py "Stateless tests bugfix validate check" "$KILL_TIMEOUT" \ python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \
--validate-bugfix --post-commit-status=file || echo 'ignore exit code' --validate-bugfix --post-commit-status=file || echo 'ignore exit code'
python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/post_commit_status.tsv" "${TEMP_PATH}/integration/post_commit_status.tsv" python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/post_commit_status.tsv" "${TEMP_PATH}/integration/post_commit_status.tsv"
@ -1895,7 +1898,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_release TEMP_PATH=${{runner.temp}}/stateful_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (release, actions) CHECK_NAME=Stateful tests (release)
REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1932,7 +1935,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_release TEMP_PATH=${{runner.temp}}/stateful_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (aarch64, actions) CHECK_NAME=Stateful tests (aarch64)
REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1969,7 +1972,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_debug TEMP_PATH=${{runner.temp}}/stateful_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (address, actions) CHECK_NAME=Stateful tests (address)
REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -2006,7 +2009,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_tsan TEMP_PATH=${{runner.temp}}/stateful_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (thread, actions) CHECK_NAME=Stateful tests (thread)
REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -2043,7 +2046,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_msan TEMP_PATH=${{runner.temp}}/stateful_msan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (memory, actions) CHECK_NAME=Stateful tests (memory)
REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -2080,7 +2083,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_ubsan TEMP_PATH=${{runner.temp}}/stateful_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (ubsan, actions) CHECK_NAME=Stateful tests (ubsan)
REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -2117,7 +2120,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_debug TEMP_PATH=${{runner.temp}}/stateful_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (debug, actions) CHECK_NAME=Stateful tests (debug)
REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -2157,7 +2160,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_thread TEMP_PATH=${{runner.temp}}/stress_thread
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (address, actions) CHECK_NAME=Stress test (address)
REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2197,7 +2200,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_thread TEMP_PATH=${{runner.temp}}/stress_thread
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (thread, actions) CHECK_NAME=Stress test (thread)
REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2233,7 +2236,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_memory TEMP_PATH=${{runner.temp}}/stress_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (memory, actions) CHECK_NAME=Stress test (memory)
REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2269,7 +2272,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_undefined TEMP_PATH=${{runner.temp}}/stress_undefined
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (undefined, actions) CHECK_NAME=Stress test (undefined)
REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2305,7 +2308,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_debug TEMP_PATH=${{runner.temp}}/stress_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (debug, actions) CHECK_NAME=Stress test (debug)
REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2344,7 +2347,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=AST fuzzer (ASan, actions) CHECK_NAME=AST fuzzer (ASan)
REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2380,7 +2383,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=AST fuzzer (TSan, actions) CHECK_NAME=AST fuzzer (TSan)
REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2416,7 +2419,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=AST fuzzer (UBSan, actions) CHECK_NAME=AST fuzzer (UBSan)
REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2452,7 +2455,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=AST fuzzer (MSan, actions) CHECK_NAME=AST fuzzer (MSan)
REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2488,7 +2491,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=AST fuzzer (debug, actions) CHECK_NAME=AST fuzzer (debug)
REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2527,7 +2530,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, actions) CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=3 RUN_BY_HASH_TOTAL=3
@ -2565,7 +2568,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, actions) CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=3 RUN_BY_HASH_TOTAL=3
@ -2603,7 +2606,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, actions) CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=3 RUN_BY_HASH_TOTAL=3
@ -2641,7 +2644,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -2679,7 +2682,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -2717,7 +2720,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -2755,7 +2758,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=3 RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -2793,7 +2796,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_release TEMP_PATH=${{runner.temp}}/integration_tests_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (release, actions) CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=2 RUN_BY_HASH_TOTAL=2
@ -2831,7 +2834,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_release TEMP_PATH=${{runner.temp}}/integration_tests_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (release, actions) CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=2 RUN_BY_HASH_TOTAL=2
@ -2869,7 +2872,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests flaky check (asan, actions) CHECK_NAME=Integration tests flaky check (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2908,7 +2911,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/unit_tests_asan TEMP_PATH=${{runner.temp}}/unit_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Unit tests (asan, actions) CHECK_NAME=Unit tests (asan)
REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2944,7 +2947,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/unit_tests_asan TEMP_PATH=${{runner.temp}}/unit_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Unit tests (release-clang, actions) CHECK_NAME=Unit tests (release-clang)
REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -2980,7 +2983,7 @@ jobs:
# cat >> "$GITHUB_ENV" << 'EOF' # cat >> "$GITHUB_ENV" << 'EOF'
# TEMP_PATH=${{runner.temp}}/unit_tests_asan # TEMP_PATH=${{runner.temp}}/unit_tests_asan
# REPORTS_PATH=${{runner.temp}}/reports_dir # REPORTS_PATH=${{runner.temp}}/reports_dir
# CHECK_NAME=Unit tests (release-gcc, actions) # CHECK_NAME=Unit tests (release-gcc)
# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse # REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse
# EOF # EOF
# - name: Download json reports # - name: Download json reports
@ -3016,7 +3019,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/unit_tests_tsan TEMP_PATH=${{runner.temp}}/unit_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Unit tests (tsan, actions) CHECK_NAME=Unit tests (tsan)
REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -3052,7 +3055,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/unit_tests_msan TEMP_PATH=${{runner.temp}}/unit_tests_msan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Unit tests (msan, actions) CHECK_NAME=Unit tests (msan)
REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -3088,7 +3091,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/unit_tests_ubsan TEMP_PATH=${{runner.temp}}/unit_tests_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Unit tests (ubsan, actions) CHECK_NAME=Unit tests (ubsan)
REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports

View File

@ -473,7 +473,7 @@ jobs:
- name: Set envs - name: Set envs
run: | run: |
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
CHECK_NAME=ClickHouse build check (actions) CHECK_NAME=ClickHouse build check
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
TEMP_PATH=${{runner.temp}}/report_check TEMP_PATH=${{runner.temp}}/report_check
@ -517,7 +517,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, actions) CHECK_NAME=Stateless tests (release)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -554,7 +554,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_release TEMP_PATH=${{runner.temp}}/stateless_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (aarch64, actions) CHECK_NAME=Stateless tests (aarch64)
REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -591,7 +591,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (address, actions) CHECK_NAME=Stateless tests (address)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -630,7 +630,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (address, actions) CHECK_NAME=Stateless tests (address)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -669,7 +669,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (thread, actions) CHECK_NAME=Stateless tests (thread)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -708,7 +708,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (thread, actions) CHECK_NAME=Stateless tests (thread)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -747,7 +747,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (thread, actions) CHECK_NAME=Stateless tests (thread)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
@ -786,7 +786,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_ubsan TEMP_PATH=${{runner.temp}}/stateless_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (ubsan, actions) CHECK_NAME=Stateless tests (ubsan)
REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
EOF EOF
@ -823,7 +823,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (memory, actions) CHECK_NAME=Stateless tests (memory)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -862,7 +862,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (memory, actions) CHECK_NAME=Stateless tests (memory)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -901,7 +901,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (memory, actions) CHECK_NAME=Stateless tests (memory)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
@ -940,7 +940,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug, actions) CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
@ -979,7 +979,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug, actions) CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
@ -1018,7 +1018,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug, actions) CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800 KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
@ -1060,7 +1060,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_debug TEMP_PATH=${{runner.temp}}/stateful_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (release, actions) CHECK_NAME=Stateful tests (release)
REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1097,7 +1097,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_release TEMP_PATH=${{runner.temp}}/stateful_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (aarch64, actions) CHECK_NAME=Stateful tests (aarch64)
REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1134,7 +1134,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_debug TEMP_PATH=${{runner.temp}}/stateful_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (address, actions) CHECK_NAME=Stateful tests (address)
REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1171,7 +1171,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_tsan TEMP_PATH=${{runner.temp}}/stateful_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (thread, actions) CHECK_NAME=Stateful tests (thread)
REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1208,7 +1208,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_msan TEMP_PATH=${{runner.temp}}/stateful_msan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (memory, actions) CHECK_NAME=Stateful tests (memory)
REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1245,7 +1245,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_ubsan TEMP_PATH=${{runner.temp}}/stateful_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (ubsan, actions) CHECK_NAME=Stateful tests (ubsan)
REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1282,7 +1282,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateful_debug TEMP_PATH=${{runner.temp}}/stateful_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateful tests (debug, actions) CHECK_NAME=Stateful tests (debug)
REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
KILL_TIMEOUT=3600 KILL_TIMEOUT=3600
EOF EOF
@ -1322,7 +1322,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_thread TEMP_PATH=${{runner.temp}}/stress_thread
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (address, actions) CHECK_NAME=Stress test (address)
REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -1362,7 +1362,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_thread TEMP_PATH=${{runner.temp}}/stress_thread
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (thread, actions) CHECK_NAME=Stress test (thread)
REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -1398,7 +1398,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_memory TEMP_PATH=${{runner.temp}}/stress_memory
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (memory, actions) CHECK_NAME=Stress test (memory)
REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -1434,7 +1434,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_undefined TEMP_PATH=${{runner.temp}}/stress_undefined
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (undefined, actions) CHECK_NAME=Stress test (undefined)
REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -1470,7 +1470,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stress_debug TEMP_PATH=${{runner.temp}}/stress_debug
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stress test (debug, actions) CHECK_NAME=Stress test (debug)
REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse
EOF EOF
- name: Download json reports - name: Download json reports
@ -1509,7 +1509,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, actions) CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=3 RUN_BY_HASH_TOTAL=3
@ -1547,7 +1547,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, actions) CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=3 RUN_BY_HASH_TOTAL=3
@ -1585,7 +1585,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, actions) CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=3 RUN_BY_HASH_TOTAL=3
@ -1623,7 +1623,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -1661,7 +1661,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -1699,7 +1699,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=2 RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -1737,7 +1737,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (thread, actions) CHECK_NAME=Integration tests (thread)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=3 RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=4 RUN_BY_HASH_TOTAL=4
@ -1775,7 +1775,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_release TEMP_PATH=${{runner.temp}}/integration_tests_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (release, actions) CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM=0 RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=2 RUN_BY_HASH_TOTAL=2
@ -1813,7 +1813,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF' cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_release TEMP_PATH=${{runner.temp}}/integration_tests_release
REPORTS_PATH=${{runner.temp}}/reports_dir REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (release, actions) CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM=1 RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=2 RUN_BY_HASH_TOTAL=2

9
.gitmodules vendored
View File

@ -259,6 +259,9 @@
[submodule "contrib/minizip-ng"] [submodule "contrib/minizip-ng"]
path = contrib/minizip-ng path = contrib/minizip-ng
url = https://github.com/zlib-ng/minizip-ng url = https://github.com/zlib-ng/minizip-ng
[submodule "contrib/qpl"]
path = contrib/qpl
url = https://github.com/intel/qpl.git
[submodule "contrib/wyhash"] [submodule "contrib/wyhash"]
path = contrib/wyhash path = contrib/wyhash
url = https://github.com/wangyi-fudan/wyhash.git url = https://github.com/wangyi-fudan/wyhash.git
@ -274,6 +277,6 @@
[submodule "contrib/liburing"] [submodule "contrib/liburing"]
path = contrib/liburing path = contrib/liburing
url = https://github.com/axboe/liburing.git url = https://github.com/axboe/liburing.git
[submodule "contrib/base-x"] [submodule "contrib/c-ares"]
path = contrib/base-x path = contrib/c-ares
url = https://github.com/ClickHouse/base-x.git url = https://github.com/ClickHouse/c-ares

View File

@ -1,4 +1,5 @@
### Table of Contents ### Table of Contents
**[ClickHouse release v22.7, 2022-07-21](#226)**<br>
**[ClickHouse release v22.6, 2022-06-16](#226)**<br> **[ClickHouse release v22.6, 2022-06-16](#226)**<br>
**[ClickHouse release v22.5, 2022-05-19](#225)**<br> **[ClickHouse release v22.5, 2022-05-19](#225)**<br>
**[ClickHouse release v22.4, 2022-04-20](#224)**<br> **[ClickHouse release v22.4, 2022-04-20](#224)**<br>
@ -7,6 +8,173 @@
**[ClickHouse release v22.1, 2022-01-18](#221)**<br> **[ClickHouse release v22.1, 2022-01-18](#221)**<br>
**[Changelog for 2021](https://clickhouse.com/docs/en/whats-new/changelog/2021/)**<br> **[Changelog for 2021](https://clickhouse.com/docs/en/whats-new/changelog/2021/)**<br>
### <a id="227"></a> ClickHouse release 22.7, 2022-07-21
#### Upgrade Notes
* Enable setting `enable_positional_arguments` by default. It allows queries like `SELECT ... ORDER BY 1, 2` where 1, 2 are the references to the select clause. If you need to return the old behavior, disable this setting. [#38204](https://github.com/ClickHouse/ClickHouse/pull/38204) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Disable `format_csv_allow_single_quotes` by default. See [#37096](https://github.com/ClickHouse/ClickHouse/issues/37096). ([Kruglov Pavel](https://github.com/Avogar)).
* `Ordinary` database engine and old storage definition syntax for `*MergeTree` tables are deprecated. By default it's not possible to create new databases with `Ordinary` engine. If `system` database has `Ordinary` engine it will be automatically converted to `Atomic` on server startup. There are settings to keep old behavior (`allow_deprecated_database_ordinary` and `allow_deprecated_syntax_for_merge_tree`), but these settings may be removed in future releases. [#38335](https://github.com/ClickHouse/ClickHouse/pull/38335) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Force rewriting comma join to inner by default (set default value `cross_to_inner_join_rewrite = 2`). To have old behavior set `cross_to_inner_join_rewrite = 1`. [#39326](https://github.com/ClickHouse/ClickHouse/pull/39326) ([Vladimir C](https://github.com/vdimir)). If you will face any incompatibilities, you can turn this setting back.
#### New Feature
* Support expressions with window functions. Closes [#19857](https://github.com/ClickHouse/ClickHouse/issues/19857). [#37848](https://github.com/ClickHouse/ClickHouse/pull/37848) ([Dmitry Novik](https://github.com/novikd)).
* Add new `direct` join algorithm for `EmbeddedRocksDB` tables, see [#33582](https://github.com/ClickHouse/ClickHouse/issues/33582). [#35363](https://github.com/ClickHouse/ClickHouse/pull/35363) ([Vladimir C](https://github.com/vdimir)).
* Added full sorting merge join algorithm. [#35796](https://github.com/ClickHouse/ClickHouse/pull/35796) ([Vladimir C](https://github.com/vdimir)).
* Implement NATS table engine, which allows to pub/sub to NATS. Closes [#32388](https://github.com/ClickHouse/ClickHouse/issues/32388). [#37171](https://github.com/ClickHouse/ClickHouse/pull/37171) ([tchepavel](https://github.com/tchepavel)). ([Kseniia Sumarokova](https://github.com/kssenii))
* Implement table function `mongodb`. Allow writes into `MongoDB` storage / table function. [#37213](https://github.com/ClickHouse/ClickHouse/pull/37213) ([aaapetrenko](https://github.com/aaapetrenko)). ([Kseniia Sumarokova](https://github.com/kssenii))
* Add `SQLInsert` output format. Closes [#38441](https://github.com/ClickHouse/ClickHouse/issues/38441). [#38477](https://github.com/ClickHouse/ClickHouse/pull/38477) ([Kruglov Pavel](https://github.com/Avogar)).
* Introduced settings `additional_table_filters`. Using this setting, you can specify additional filtering condition for a table which will be applied directly after reading. Example: `select number, x, y from (select number from system.numbers limit 5) f any left join (select x, y from table_1) s on f.number = s.x settings additional_table_filters={'system.numbers : 'number != 3', 'table_1' : 'x != 2'}`. Introduced setting `additional_result_filter` which specifies additional filtering condition for query result. Closes [#37918](https://github.com/ClickHouse/ClickHouse/issues/37918). [#38475](https://github.com/ClickHouse/ClickHouse/pull/38475) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Add `compatibility` setting and `system.settings_changes` system table that contains information about changes in settings through ClickHouse versions. Closes [#35972](https://github.com/ClickHouse/ClickHouse/issues/35972). [#38957](https://github.com/ClickHouse/ClickHouse/pull/38957) ([Kruglov Pavel](https://github.com/Avogar)).
* Add functions `translate(string, from_string, to_string)` and `translateUTF8(string, from_string, to_string)`. It translates some characters to another. [#38935](https://github.com/ClickHouse/ClickHouse/pull/38935) ([Nikolay Degterinsky](https://github.com/evillique)).
* Support `parseTimeDelta` function. It can be used like ` ;-+,:` can be used as separators, eg. `1yr-2mo`, `2m:6s`: `SELECT parseTimeDelta('1yr-2mo-4w + 12 days, 3 hours : 1 minute ; 33 seconds')`. [#39071](https://github.com/ClickHouse/ClickHouse/pull/39071) ([jiahui-97](https://github.com/jiahui-97)).
* Added `CREATE TABLE ... EMPTY AS SELECT` query. It automatically deduces table structure from the SELECT query, but does not fill the table after creation. Resolves [#38049](https://github.com/ClickHouse/ClickHouse/issues/38049). [#38272](https://github.com/ClickHouse/ClickHouse/pull/38272) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Added options to limit IO operations with remote storage: `max_remote_read_network_bandwidth_for_server` and `max_remote_write_network_bandwidth_for_server`. [#39095](https://github.com/ClickHouse/ClickHouse/pull/39095) ([Sergei Trifonov](https://github.com/serxa)).
* Add `group_by_use_nulls` setting to make aggregation key columns nullable in the case of ROLLUP, CUBE and GROUPING SETS. Closes [#37359](https://github.com/ClickHouse/ClickHouse/issues/37359). [#38642](https://github.com/ClickHouse/ClickHouse/pull/38642) ([Dmitry Novik](https://github.com/novikd)).
* Add the ability to specify compression level during data export. [#38907](https://github.com/ClickHouse/ClickHouse/pull/38907) ([Nikolay Degterinsky](https://github.com/evillique)).
* Add an option to require explicit grants to SELECT from the `system` database. Details: [#38970](https://github.com/ClickHouse/ClickHouse/pull/38970) ([Vitaly Baranov](https://github.com/vitlibar)).
* Functions `multiMatchAny`, `multiMatchAnyIndex`, `multiMatchAllIndices` and their fuzzy variants now accept non-const pattern array argument. [#38485](https://github.com/ClickHouse/ClickHouse/pull/38485) ([Robert Schulze](https://github.com/rschu1ze)). SQL function `multiSearchAllPositions` now accepts non-const needle arguments. [#39167](https://github.com/ClickHouse/ClickHouse/pull/39167) ([Robert Schulze](https://github.com/rschu1ze)).
* Add a setting `zstd_window_log_max` to configure max memory usage on zstd decoding when importing external files. Closes [#35693](https://github.com/ClickHouse/ClickHouse/issues/35693). [#37015](https://github.com/ClickHouse/ClickHouse/pull/37015) ([wuxiaobai24](https://github.com/wuxiaobai24)).
* Add `send_logs_source_regexp` setting. Send server text logs with specified regexp to match log source name. Empty means all sources. [#39161](https://github.com/ClickHouse/ClickHouse/pull/39161) ([Amos Bird](https://github.com/amosbird)).
* Support `ALTER` for `Hive` tables. [#38214](https://github.com/ClickHouse/ClickHouse/pull/38214) ([lgbo](https://github.com/lgbo-ustc)).
* Support `isNullable` function. This function checks whether it's argument is nullable and return 1 or 0. Closes [#38611](https://github.com/ClickHouse/ClickHouse/issues/38611). [#38841](https://github.com/ClickHouse/ClickHouse/pull/38841) ([lokax](https://github.com/lokax)).
* Added functions for base58 encoding/decoding. [#38159](https://github.com/ClickHouse/ClickHouse/pull/38159) ([Andrey Zvonov](https://github.com/zvonand)).
* Add chart visualization to Play UI. [#38197](https://github.com/ClickHouse/ClickHouse/pull/38197) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Added L2 Squared distance and norm functions for both arrays and tuples. [#38545](https://github.com/ClickHouse/ClickHouse/pull/38545) ([Julian Gilyadov](https://github.com/israelg99)).
* Add ability to pass HTTP headers to the `url` table function / storage via SQL. Closes [#37897](https://github.com/ClickHouse/ClickHouse/issues/37897). [#38176](https://github.com/ClickHouse/ClickHouse/pull/38176) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Add `clickhouse-diagnostics` binary to the packages. [#38647](https://github.com/ClickHouse/ClickHouse/pull/38647) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### Experimental Feature
* Adds new setting `implicit_transaction` to run standalone queries inside a transaction. It handles both creation and closing (via COMMIT if the query succeeded or ROLLBACK if it didn't) of the transaction automatically. [#38344](https://github.com/ClickHouse/ClickHouse/pull/38344) ([Raúl Marín](https://github.com/Algunenano)).
#### Performance Improvement
* Distinct optimization for sorted columns. Use specialized distinct transformation in case input stream is sorted by column(s) in distinct. Optimization can be applied to pre-distinct, final distinct, or both. Initial implementation by @dimarub2000. [#37803](https://github.com/ClickHouse/ClickHouse/pull/37803) ([Igor Nikonov](https://github.com/devcrafter)).
* Improve performance of `ORDER BY`, `MergeTree` merges, window functions using batch version of `BinaryHeap`. [#38022](https://github.com/ClickHouse/ClickHouse/pull/38022) ([Maksim Kita](https://github.com/kitaisreal)).
* More parallel execution for queries with `FINAL` [#36396](https://github.com/ClickHouse/ClickHouse/pull/36396) ([Nikita Taranov](https://github.com/nickitat)).
* Fix significant join performance regression which was introduced in [#35616](https://github.com/ClickHouse/ClickHouse/pull/35616). It's interesting that common join queries such as ssb queries have been 10 times slower for almost 3 months while no one complains. [#38052](https://github.com/ClickHouse/ClickHouse/pull/38052) ([Amos Bird](https://github.com/amosbird)).
* Migrate from the Intel hyperscan library to vectorscan, this speeds up many string matching on non-x86 platforms. [#38171](https://github.com/ClickHouse/ClickHouse/pull/38171) ([Robert Schulze](https://github.com/rschu1ze)).
* Increased parallelism of query plan steps executed after aggregation. [#38295](https://github.com/ClickHouse/ClickHouse/pull/38295) ([Nikita Taranov](https://github.com/nickitat)).
* Improve performance of insertion to columns of type `JSON`. [#38320](https://github.com/ClickHouse/ClickHouse/pull/38320) ([Anton Popov](https://github.com/CurtizJ)).
* Optimized insertion and lookups in the HashTable. [#38413](https://github.com/ClickHouse/ClickHouse/pull/38413) ([Nikita Taranov](https://github.com/nickitat)).
* Fix performance degradation from [#32493](https://github.com/ClickHouse/ClickHouse/issues/32493). [#38417](https://github.com/ClickHouse/ClickHouse/pull/38417) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Improve performance of joining with numeric columns using SIMD instructions. [#37235](https://github.com/ClickHouse/ClickHouse/pull/37235) ([zzachimed](https://github.com/zzachimed)). [#38565](https://github.com/ClickHouse/ClickHouse/pull/38565) ([Maksim Kita](https://github.com/kitaisreal)).
* Norm and Distance functions for arrays speed up 1.2-2 times. [#38740](https://github.com/ClickHouse/ClickHouse/pull/38740) ([Alexander Gololobov](https://github.com/davenger)).
* Add AVX-512 VBMI optimized `copyOverlap32Shuffle` for LZ4 decompression. In other words, LZ4 decompression performance is improved. [#37891](https://github.com/ClickHouse/ClickHouse/pull/37891) ([Guo Wangyang](https://github.com/guowangy)).
* `ORDER BY (a, b)` will use all the same benefits as `ORDER BY a, b`. [#38873](https://github.com/ClickHouse/ClickHouse/pull/38873) ([Igor Nikonov](https://github.com/devcrafter)).
* Align branches within a 32B boundary to make benchmark more stable. [#38988](https://github.com/ClickHouse/ClickHouse/pull/38988) ([Guo Wangyang](https://github.com/guowangy)). It improves performance 1..2% on average for Intel.
* Executable UDF, executable dictionaries, and Executable tables will avoid wasting one second during wait for subprocess termination. [#38929](https://github.com/ClickHouse/ClickHouse/pull/38929) ([Constantine Peresypkin](https://github.com/pkit)).
* Optimize accesses to `system.stack_trace` table if not all columns are selected. [#39177](https://github.com/ClickHouse/ClickHouse/pull/39177) ([Azat Khuzhin](https://github.com/azat)).
* Improve isNullable/isConstant/isNull/isNotNull performance for LowCardinality argument. [#39192](https://github.com/ClickHouse/ClickHouse/pull/39192) ([Kruglov Pavel](https://github.com/Avogar)).
* Optimized processing of ORDER BY in window functions. [#34632](https://github.com/ClickHouse/ClickHouse/pull/34632) ([Vladimir Chebotarev](https://github.com/excitoon)).
* The table `system.asynchronous_metric_log` is further optimized for storage space. This closes [#38134](https://github.com/ClickHouse/ClickHouse/issues/38134). See the [YouTube video](https://www.youtube.com/watch?v=0fSp9SF8N8A). [#38428](https://github.com/ClickHouse/ClickHouse/pull/38428) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
#### Improvement
* Support SQL standard CREATE INDEX and DROP INDEX syntax. [#35166](https://github.com/ClickHouse/ClickHouse/pull/35166) ([Jianmei Zhang](https://github.com/zhangjmruc)).
* Send profile events for INSERT queries (previously only SELECT was supported). [#37391](https://github.com/ClickHouse/ClickHouse/pull/37391) ([Azat Khuzhin](https://github.com/azat)).
* Implement in order aggregation (`optimize_aggregation_in_order`) for fully materialized projections. [#37469](https://github.com/ClickHouse/ClickHouse/pull/37469) ([Azat Khuzhin](https://github.com/azat)).
* Remove subprocess run for kerberos initialization. Added new integration test. Closes [#27651](https://github.com/ClickHouse/ClickHouse/issues/27651). [#38105](https://github.com/ClickHouse/ClickHouse/pull/38105) ([Roman Vasin](https://github.com/rvasin)).
* * Add setting `multiple_joins_try_to_keep_original_names` to not rewrite identifier name on multiple JOINs rewrite, close [#34697](https://github.com/ClickHouse/ClickHouse/issues/34697). [#38149](https://github.com/ClickHouse/ClickHouse/pull/38149) ([Vladimir C](https://github.com/vdimir)).
* Improved trace-visualizer UX. [#38169](https://github.com/ClickHouse/ClickHouse/pull/38169) ([Sergei Trifonov](https://github.com/serxa)).
* Enable stack trace collection and query profiler for AArch64. [#38181](https://github.com/ClickHouse/ClickHouse/pull/38181) ([Maksim Kita](https://github.com/kitaisreal)).
* Do not skip symlinks in `user_defined` directory during SQL user defined functions loading. Closes [#38042](https://github.com/ClickHouse/ClickHouse/issues/38042). [#38184](https://github.com/ClickHouse/ClickHouse/pull/38184) ([Maksim Kita](https://github.com/kitaisreal)).
* Added background cleanup of subdirectories in `store/`. In some cases clickhouse-server might left garbage subdirectories in `store/` (for example, on unsuccessful table creation) and those dirs were never been removed. Fixes [#33710](https://github.com/ClickHouse/ClickHouse/issues/33710). [#38265](https://github.com/ClickHouse/ClickHouse/pull/38265) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Add `DESCRIBE CACHE` query to show cache settings from config. Add `SHOW CACHES` query to show available filesystem caches list. [#38279](https://github.com/ClickHouse/ClickHouse/pull/38279) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Add access check for `system drop filesystem cache`. Support ON CLUSTER. [#38319](https://github.com/ClickHouse/ClickHouse/pull/38319) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix PostgreSQL database engine incompatibility on upgrade from 21.3 to 22.3. Closes [#36659](https://github.com/ClickHouse/ClickHouse/issues/36659). [#38369](https://github.com/ClickHouse/ClickHouse/pull/38369) ([Kseniia Sumarokova](https://github.com/kssenii)).
* `filesystemAvailable` and similar functions now work in `clickhouse-local`. This closes [#38423](https://github.com/ClickHouse/ClickHouse/issues/38423). [#38424](https://github.com/ClickHouse/ClickHouse/pull/38424) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Add `revision` function. [#38555](https://github.com/ClickHouse/ClickHouse/pull/38555) ([Azat Khuzhin](https://github.com/azat)).
* Fix GCS via proxy tunnel usage. [#38726](https://github.com/ClickHouse/ClickHouse/pull/38726) ([Azat Khuzhin](https://github.com/azat)).
* Support `\i file` in clickhouse client / local (similar to psql \i). [#38813](https://github.com/ClickHouse/ClickHouse/pull/38813) ([Kseniia Sumarokova](https://github.com/kssenii)).
* New option `optimize = 1` in `EXPLAIN AST`. If enabled, it shows AST after it's rewritten, otherwise AST of original query. Disabled by default. [#38910](https://github.com/ClickHouse/ClickHouse/pull/38910) ([Igor Nikonov](https://github.com/devcrafter)).
* Allow trailing comma in columns list. closes [#38425](https://github.com/ClickHouse/ClickHouse/issues/38425). [#38440](https://github.com/ClickHouse/ClickHouse/pull/38440) ([chen](https://github.com/xiedeyantu)).
* Bugfixes and performance improvements for `parallel_hash` JOIN method. [#37648](https://github.com/ClickHouse/ClickHouse/pull/37648) ([Vladimir C](https://github.com/vdimir)).
* Support hadoop secure RPC transfer (hadoop.rpc.protection=privacy and hadoop.rpc.protection=integrity). [#37852](https://github.com/ClickHouse/ClickHouse/pull/37852) ([Peng Liu](https://github.com/michael1589)).
* Add struct type support in `StorageHive`. [#38118](https://github.com/ClickHouse/ClickHouse/pull/38118) ([lgbo](https://github.com/lgbo-ustc)).
* S3 single objects are now removed with `RemoveObjectRequest`. Implement compatibility with GCP which did not allow to use `removeFileIfExists` effectively breaking approximately half of `remove` functionality. Automatic detection for `DeleteObjects` S3 API, that is not supported by GCS. This will allow to use GCS without explicit `support_batch_delete=0` in configuration. [#37882](https://github.com/ClickHouse/ClickHouse/pull/37882) ([Vladimir Chebotarev](https://github.com/excitoon)).
* Expose basic ClickHouse Keeper related monitoring data (via ProfileEvents and CurrentMetrics). [#38072](https://github.com/ClickHouse/ClickHouse/pull/38072) ([lingpeng0314](https://github.com/lingpeng0314)).
* Support `auto_close` option for PostgreSQL engine connection. Closes [#31486](https://github.com/ClickHouse/ClickHouse/issues/31486). [#38363](https://github.com/ClickHouse/ClickHouse/pull/38363) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Allow `NULL` modifier in columns declaration for table functions. [#38816](https://github.com/ClickHouse/ClickHouse/pull/38816) ([Kruglov Pavel](https://github.com/Avogar)).
* Deactivate `mutations_finalizing_task` before shutdown to avoid benign `TABLE_IS_READ_ONLY` errors during shutdown. [#38851](https://github.com/ClickHouse/ClickHouse/pull/38851) ([Raúl Marín](https://github.com/Algunenano)).
* Eliminate unnecessary waiting of SELECT queries after ALTER queries in presence of INSERT queries if you use deprecated Ordinary databases. [#38864](https://github.com/ClickHouse/ClickHouse/pull/38864) ([Azat Khuzhin](https://github.com/azat)).
* New option `rewrite` in `EXPLAIN AST`. If enabled, it shows AST after it's rewritten, otherwise AST of original query. Disabled by default. [#38910](https://github.com/ClickHouse/ClickHouse/pull/38910) ([Igor Nikonov](https://github.com/devcrafter)).
* Stop reporting Zookeeper "Node exists" exceptions in system.errors when they are expected. [#38961](https://github.com/ClickHouse/ClickHouse/pull/38961) ([Raúl Marín](https://github.com/Algunenano)).
* `clickhouse-keeper`: add support for real-time digest calculation and verification. It is disabled by default. [#37555](https://github.com/ClickHouse/ClickHouse/pull/37555) ([Antonio Andelic](https://github.com/antonio2368)).
* Allow to specify globs `* or {expr1, expr2, expr3}` inside a key for `clickhouse-extract-from-config` tool. [#38966](https://github.com/ClickHouse/ClickHouse/pull/38966) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
* clearOldLogs: Don't report KEEPER_EXCEPTION on concurrent deletes. [#39016](https://github.com/ClickHouse/ClickHouse/pull/39016) ([Raúl Marín](https://github.com/Algunenano)).
* clickhouse-keeper improvement: persist meta-information about keeper servers to disk. [#39069](https://github.com/ClickHouse/ClickHouse/pull/39069) ([Antonio Andelic](https://github.com/antonio2368)). This will make it easier to operate if you shutdown or restart all keeper nodes at the same time.
* Continue without exception when running out of disk space when using filesystem cache. [#39106](https://github.com/ClickHouse/ClickHouse/pull/39106) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Handling SIGTERM signals from k8s. [#39130](https://github.com/ClickHouse/ClickHouse/pull/39130) ([Timur Solodovnikov](https://github.com/tsolodov)).
* Add `merge_algorithm` column (Undecided, Horizontal, Vertical) to system.part_log. [#39181](https://github.com/ClickHouse/ClickHouse/pull/39181) ([Azat Khuzhin](https://github.com/azat)).
* Don't increment a counter in `system.errors` when the disk is not rotational. [#39216](https://github.com/ClickHouse/ClickHouse/pull/39216) ([Raúl Marín](https://github.com/Algunenano)).
* The metric `result_bytes` for `INSERT` queries in `system.query_log` shows number of bytes inserted. Previously value was incorrect and stored the same value as `result_rows`. [#39225](https://github.com/ClickHouse/ClickHouse/pull/39225) ([Ilya Yatsishin](https://github.com/qoega)).
* The CPU usage metric in clickhouse-client will be displayed in a better way. Fixes [#38756](https://github.com/ClickHouse/ClickHouse/issues/38756). [#39280](https://github.com/ClickHouse/ClickHouse/pull/39280) ([Sergei Trifonov](https://github.com/serxa)).
* Rethrow exception on filesystem cache initialization on server startup, better error message. [#39386](https://github.com/ClickHouse/ClickHouse/pull/39386) ([Kseniia Sumarokova](https://github.com/kssenii)).
* OpenTelemetry now collects traces without Processors spans by default (there are too many). To enable Processors spans collection `opentelemetry_trace_processors` setting. [#39170](https://github.com/ClickHouse/ClickHouse/pull/39170) ([Ilya Yatsishin](https://github.com/qoega)).
* Functions `multiMatch[Fuzzy](AllIndices/Any/AnyIndex)` - don't throw a logical error if the needle argument is empty. [#39012](https://github.com/ClickHouse/ClickHouse/pull/39012) ([Robert Schulze](https://github.com/rschu1ze)).
* Allow to declare `RabbitMQ` queue without default arguments `x-max-length` and `x-overflow`. [#39259](https://github.com/ClickHouse/ClickHouse/pull/39259) ([rnbondarenko](https://github.com/rnbondarenko)).
#### Build/Testing/Packaging Improvement
* Apply Clang Thread Safety Analysis (TSA) annotations to ClickHouse. [#38068](https://github.com/ClickHouse/ClickHouse/pull/38068) ([Robert Schulze](https://github.com/rschu1ze)).
* Adapt universal installation script for FreeBSD. [#39302](https://github.com/ClickHouse/ClickHouse/pull/39302) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Preparation for building on `s390x` platform. [#39193](https://github.com/ClickHouse/ClickHouse/pull/39193) ([Harry Lee](https://github.com/HarryLeeIBM)).
* Fix a bug in `jemalloc` library [#38757](https://github.com/ClickHouse/ClickHouse/pull/38757) ([Azat Khuzhin](https://github.com/azat)).
* Hardware benchmark now has support for automatic results uploading. [#38427](https://github.com/ClickHouse/ClickHouse/pull/38427) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* System table "system.licenses" is now correctly populated on Mac (Darwin). [#38294](https://github.com/ClickHouse/ClickHouse/pull/38294) ([Robert Schulze](https://github.com/rschu1ze)).
* Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
* Fix rounding for `Decimal128/Decimal256` with more than 19-digits long scale. [#38027](https://github.com/ClickHouse/ClickHouse/pull/38027) ([Igor Nikonov](https://github.com/devcrafter)).
* Fixed crash caused by data race in storage `Hive` (integration table engine). [#38887](https://github.com/ClickHouse/ClickHouse/pull/38887) ([lgbo](https://github.com/lgbo-ustc)).
* Fix crash when executing GRANT ALL ON *.* with ON CLUSTER. It was broken in https://github.com/ClickHouse/ClickHouse/pull/35767. This closes [#38618](https://github.com/ClickHouse/ClickHouse/issues/38618). [#38674](https://github.com/ClickHouse/ClickHouse/pull/38674) ([Vitaly Baranov](https://github.com/vitlibar)).
* Correct glob expansion in case of `{0..10}` forms. Fixes [#38498](https://github.com/ClickHouse/ClickHouse/issues/38498) Current Implementation is similar to what shell does mentiond by @rschu1ze [here](https://github.com/ClickHouse/ClickHouse/pull/38502#issuecomment-1169057723). [#38502](https://github.com/ClickHouse/ClickHouse/pull/38502) ([Heena Bansal](https://github.com/HeenaBansal2009)).
* Fix crash for `mapUpdate`, `mapFilter` functions when using with constant map argument. Closes [#38547](https://github.com/ClickHouse/ClickHouse/issues/38547). [#38553](https://github.com/ClickHouse/ClickHouse/pull/38553) ([hexiaoting](https://github.com/hexiaoting)).
* Fix `toHour` monotonicity information for query optimization which can lead to incorrect query result (incorrect index analysis). This fixes [#38333](https://github.com/ClickHouse/ClickHouse/issues/38333). [#38675](https://github.com/ClickHouse/ClickHouse/pull/38675) ([Amos Bird](https://github.com/amosbird)).
* Fix checking whether s3 storage support parallel writes. It resulted in s3 parallel writes not working. [#38792](https://github.com/ClickHouse/ClickHouse/pull/38792) ([chen](https://github.com/xiedeyantu)).
* Fix s3 seekable reads with parallel read buffer. (Affected memory usage during query). Closes [#38258](https://github.com/ClickHouse/ClickHouse/issues/38258). [#38802](https://github.com/ClickHouse/ClickHouse/pull/38802) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Update `simdjson`. This fixes [#38621](https://github.com/ClickHouse/ClickHouse/issues/38621) - a buffer overflow on machines with the latest Intel CPUs with AVX-512 VBMI. [#38838](https://github.com/ClickHouse/ClickHouse/pull/38838) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Fix possible logical error for Vertical merges. [#38859](https://github.com/ClickHouse/ClickHouse/pull/38859) ([Maksim Kita](https://github.com/kitaisreal)).
* Fix settings profile with seconds unit. [#38896](https://github.com/ClickHouse/ClickHouse/pull/38896) ([Raúl Marín](https://github.com/Algunenano)).
* Fix incorrect partition pruning when there is a nullable partition key. Note: most likely you don't use nullable partition keys - this is an obscure feature you should not use. Nullable keys are a nonsense and this feature is only needed for some crazy use-cases. This fixes [#38941](https://github.com/ClickHouse/ClickHouse/issues/38941). [#38946](https://github.com/ClickHouse/ClickHouse/pull/38946) ([Amos Bird](https://github.com/amosbird)).
* Improve `fsync_part_directory` for fetches. [#38993](https://github.com/ClickHouse/ClickHouse/pull/38993) ([Azat Khuzhin](https://github.com/azat)).
* Fix possible dealock inside `OvercommitTracker`. Fixes [#37794](https://github.com/ClickHouse/ClickHouse/issues/37794). [#39030](https://github.com/ClickHouse/ClickHouse/pull/39030) ([Dmitry Novik](https://github.com/novikd)).
* Fix bug in filesystem cache that could happen in some corner case which coincided with cache capacity hitting the limit. Closes [#39066](https://github.com/ClickHouse/ClickHouse/issues/39066). [#39070](https://github.com/ClickHouse/ClickHouse/pull/39070) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix some corner cases of interpretation of the arguments of window expressions. Fixes [#38538](https://github.com/ClickHouse/ClickHouse/issues/38538) Allow using of higher-order functions in window expressions. [#39112](https://github.com/ClickHouse/ClickHouse/pull/39112) ([Dmitry Novik](https://github.com/novikd)).
* Keep `LowCardinality` type in `tuple` function. Previously `LowCardinality` type was dropped and elements of created tuple had underlying type of `LowCardinality`. [#39113](https://github.com/ClickHouse/ClickHouse/pull/39113) ([Anton Popov](https://github.com/CurtizJ)).
* Fix error `Block structure mismatch` which could happen for INSERT into table with attached MATERIALIZED VIEW and enabled setting `extremes = 1`. Closes [#29759](https://github.com/ClickHouse/ClickHouse/issues/29759) and [#38729](https://github.com/ClickHouse/ClickHouse/issues/38729). [#39125](https://github.com/ClickHouse/ClickHouse/pull/39125) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix unexpected query result when both `optimize_trivial_count_query` and `empty_result_for_aggregation_by_empty_set` are set to true. This fixes [#39140](https://github.com/ClickHouse/ClickHouse/issues/39140). [#39155](https://github.com/ClickHouse/ClickHouse/pull/39155) ([Amos Bird](https://github.com/amosbird)).
* Fixed error `Not found column Type in block` in selects with `PREWHERE` and read-in-order optimizations. [#39157](https://github.com/ClickHouse/ClickHouse/pull/39157) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Fix extremely rare race condition in during hardlinks for remote filesystem. The only way to reproduce it is concurrent run of backups. [#39190](https://github.com/ClickHouse/ClickHouse/pull/39190) ([alesapin](https://github.com/alesapin)).
* (zero-copy replication is an experimental feature that should not be used in production) Fix fetch of in-memory part with `allow_remote_fs_zero_copy_replication`. [#39214](https://github.com/ClickHouse/ClickHouse/pull/39214) ([Azat Khuzhin](https://github.com/azat)).
* (MaterializedPostgreSQL - experimental feature). Fix segmentation fault in MaterializedPostgreSQL database engine, which could happen if some exception occurred at replication initialisation. Closes [#36939](https://github.com/ClickHouse/ClickHouse/issues/36939). [#39272](https://github.com/ClickHouse/ClickHouse/pull/39272) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix incorrect fetch of table metadata from PostgreSQL database engine. Closes [#33502](https://github.com/ClickHouse/ClickHouse/issues/33502). [#39283](https://github.com/ClickHouse/ClickHouse/pull/39283) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix projection exception when aggregation keys are wrapped inside other functions. This fixes [#37151](https://github.com/ClickHouse/ClickHouse/issues/37151). [#37155](https://github.com/ClickHouse/ClickHouse/pull/37155) ([Amos Bird](https://github.com/amosbird)).
* Fix possible logical error `... with argument with type Nothing and default implementation for Nothing is expected to return result with type Nothing, got ...` in some functions. Closes: [#37610](https://github.com/ClickHouse/ClickHouse/issues/37610) Closes: [#37741](https://github.com/ClickHouse/ClickHouse/issues/37741). [#37759](https://github.com/ClickHouse/ClickHouse/pull/37759) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix incorrect columns order in subqueries of UNION (in case of duplicated columns in subselects may produce incorrect result). [#37887](https://github.com/ClickHouse/ClickHouse/pull/37887) ([Azat Khuzhin](https://github.com/azat)).
* Fix incorrect work of MODIFY ALTER Column with column names that contain dots. Closes [#37907](https://github.com/ClickHouse/ClickHouse/issues/37907). [#37971](https://github.com/ClickHouse/ClickHouse/pull/37971) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix reading of sparse columns from `MergeTree` tables that store their data in S3. [#37978](https://github.com/ClickHouse/ClickHouse/pull/37978) ([Anton Popov](https://github.com/CurtizJ)).
* Fix possible crash in `Distributed` async insert in case of removing a replica from config. [#38029](https://github.com/ClickHouse/ClickHouse/pull/38029) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix "Missing columns" for GLOBAL JOIN with CTE without alias. [#38056](https://github.com/ClickHouse/ClickHouse/pull/38056) ([Azat Khuzhin](https://github.com/azat)).
* Rewrite tuple functions as literals in backwards-compatibility mode. [#38096](https://github.com/ClickHouse/ClickHouse/pull/38096) ([Anton Kozlov](https://github.com/tonickkozlov)).
* Fix redundant memory reservation for output block during `ORDER BY`. [#38127](https://github.com/ClickHouse/ClickHouse/pull/38127) ([iyupeng](https://github.com/iyupeng)).
* Fix possible logical error `Bad cast from type DB::IColumn* to DB::ColumnNullable*` in array mapped functions. Closes [#38006](https://github.com/ClickHouse/ClickHouse/issues/38006). [#38132](https://github.com/ClickHouse/ClickHouse/pull/38132) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix temporary name clash in partial merge join, close [#37928](https://github.com/ClickHouse/ClickHouse/issues/37928). [#38135](https://github.com/ClickHouse/ClickHouse/pull/38135) ([Vladimir C](https://github.com/vdimir)).
* Some minr issue with queries like `CREATE TABLE nested_name_tuples (`a` Tuple(x String, y Tuple(i Int32, j String))) ENGINE = Memory;` [#38136](https://github.com/ClickHouse/ClickHouse/pull/38136) ([lgbo](https://github.com/lgbo-ustc)).
* Fix bug with nested short-circuit functions that led to execution of arguments even if condition is false. Closes [#38040](https://github.com/ClickHouse/ClickHouse/issues/38040). [#38173](https://github.com/ClickHouse/ClickHouse/pull/38173) ([Kruglov Pavel](https://github.com/Avogar)).
* (Window View is a experimental feature) Fix LOGICAL_ERROR for WINDOW VIEW with incorrect structure. [#38205](https://github.com/ClickHouse/ClickHouse/pull/38205) ([Azat Khuzhin](https://github.com/azat)).
* Update librdkafka submodule to fix crash when an OAUTHBEARER refresh callback is set. [#38225](https://github.com/ClickHouse/ClickHouse/pull/38225) ([Rafael Acevedo](https://github.com/racevedoo)).
* Fix INSERT into Distributed hung due to ProfileEvents. [#38307](https://github.com/ClickHouse/ClickHouse/pull/38307) ([Azat Khuzhin](https://github.com/azat)).
* Fix retries in PostgreSQL engine. [#38310](https://github.com/ClickHouse/ClickHouse/pull/38310) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix optimization in PartialSortingTransform (SIGSEGV and possible incorrect result). [#38324](https://github.com/ClickHouse/ClickHouse/pull/38324) ([Azat Khuzhin](https://github.com/azat)).
* Fix RabbitMQ with formats based on PeekableReadBuffer. Closes [#38061](https://github.com/ClickHouse/ClickHouse/issues/38061). [#38356](https://github.com/ClickHouse/ClickHouse/pull/38356) ([Kseniia Sumarokova](https://github.com/kssenii)).
* MaterializedPostgreSQL - experimentail feature. Fix possible `Invalid number of rows in Chunk` in MaterializedPostgreSQL. Closes [#37323](https://github.com/ClickHouse/ClickHouse/issues/37323). [#38360](https://github.com/ClickHouse/ClickHouse/pull/38360) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix RabbitMQ configuration with connection string setting. Closes [#36531](https://github.com/ClickHouse/ClickHouse/issues/36531). [#38365](https://github.com/ClickHouse/ClickHouse/pull/38365) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix PostgreSQL engine not using PostgreSQL schema when retrieving array dimension size. Closes [#36755](https://github.com/ClickHouse/ClickHouse/issues/36755). Closes [#36772](https://github.com/ClickHouse/ClickHouse/issues/36772). [#38366](https://github.com/ClickHouse/ClickHouse/pull/38366) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix possibly incorrect result of distributed queries with `DISTINCT` and `LIMIT`. Fixes [#38282](https://github.com/ClickHouse/ClickHouse/issues/38282). [#38371](https://github.com/ClickHouse/ClickHouse/pull/38371) ([Anton Popov](https://github.com/CurtizJ)).
* Fix wrong results of countSubstrings() & position() on patterns with 0-bytes. [#38589](https://github.com/ClickHouse/ClickHouse/pull/38589) ([Robert Schulze](https://github.com/rschu1ze)).
* Now it's possible to start a clickhouse-server and attach/detach tables even for tables with the incorrect values of IPv4/IPv6 representation. Proper fix for issue [#35156](https://github.com/ClickHouse/ClickHouse/issues/35156). [#38590](https://github.com/ClickHouse/ClickHouse/pull/38590) ([alesapin](https://github.com/alesapin)).
* `rankCorr` function will work correctly if some arguments are NaNs. This closes [#38396](https://github.com/ClickHouse/ClickHouse/issues/38396). [#38722](https://github.com/ClickHouse/ClickHouse/pull/38722) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Fix `parallel_view_processing=1` with `optimize_trivial_insert_select=1`. Fix `max_insert_threads` while pushing to views. [#38731](https://github.com/ClickHouse/ClickHouse/pull/38731) ([Azat Khuzhin](https://github.com/azat)).
* Fix use-after-free for aggregate functions with `Map` combinator that leads to incorrect result. [#38748](https://github.com/ClickHouse/ClickHouse/pull/38748) ([Azat Khuzhin](https://github.com/azat)).
### <a id="226"></a> ClickHouse release 22.6, 2022-06-16 ### <a id="226"></a> ClickHouse release 22.6, 2022-06-16
#### Backward Incompatible Change #### Backward Incompatible Change

View File

@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.14) cmake_minimum_required(VERSION 3.15)
project(ClickHouse LANGUAGES C CXX ASM) project(ClickHouse LANGUAGES C CXX ASM)
@ -554,6 +554,16 @@ macro (clickhouse_add_executable target)
endif() endif()
endmacro() endmacro()
# With cross-compiling, all targets are built for the target platform which usually different from the host
# platform. This is problematic if a build artifact X (e.g. a file or an executable) is generated by running
# another executable Y previously produced in the build. This is solved by compiling and running Y for/on
# the host platform. Add target to the list:
# add_native_target(<target> ...)
set_property (GLOBAL PROPERTY NATIVE_BUILD_TARGETS)
function (add_native_target)
set_property (GLOBAL APPEND PROPERTY NATIVE_BUILD_TARGETS ${ARGV})
endfunction (add_native_target)
set(ConfigIncludePath ${CMAKE_CURRENT_BINARY_DIR}/includes/configs CACHE INTERNAL "Path to generated configuration files.") set(ConfigIncludePath ${CMAKE_CURRENT_BINARY_DIR}/includes/configs CACHE INTERNAL "Path to generated configuration files.")
include_directories(${ConfigIncludePath}) include_directories(${ConfigIncludePath})
@ -568,3 +578,33 @@ add_subdirectory (tests)
add_subdirectory (utils) add_subdirectory (utils)
include (cmake/sanitize_target_link_libraries.cmake) include (cmake/sanitize_target_link_libraries.cmake)
# Build native targets if necessary
get_property(NATIVE_BUILD_TARGETS GLOBAL PROPERTY NATIVE_BUILD_TARGETS)
if (NATIVE_BUILD_TARGETS
AND NOT(
CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR
)
)
message (STATUS "Building native targets...")
set (NATIVE_BUILD_DIR "${CMAKE_BINARY_DIR}/native")
execute_process(
COMMAND ${CMAKE_COMMAND} -E make_directory "${NATIVE_BUILD_DIR}"
COMMAND_ECHO STDOUT)
execute_process(
COMMAND ${CMAKE_COMMAND}
"-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
"-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
${CMAKE_SOURCE_DIR}
WORKING_DIRECTORY "${NATIVE_BUILD_DIR}"
COMMAND_ECHO STDOUT)
execute_process(
COMMAND ${CMAKE_COMMAND} --build "${NATIVE_BUILD_DIR}" --target ${NATIVE_BUILD_TARGETS}
COMMAND_ECHO STDOUT)
endif ()

View File

@ -669,18 +669,18 @@ std::string JSON::getName() const
return getString(); return getString();
} }
StringRef JSON::getRawString() const std::string_view JSON::getRawString() const
{ {
Pos s = ptr_begin; Pos s = ptr_begin;
if (*s != '"') if (*s != '"')
throw JSONException(std::string("JSON: expected \", got ") + *s); throw JSONException(std::string("JSON: expected \", got ") + *s);
while (++s != ptr_end && *s != '"'); while (++s != ptr_end && *s != '"');
if (s != ptr_end) if (s != ptr_end)
return StringRef(ptr_begin + 1, s - ptr_begin - 1); return std::string_view(ptr_begin + 1, s - ptr_begin - 1);
throw JSONException("JSON: incorrect syntax (expected end of string, found end of JSON)."); throw JSONException("JSON: incorrect syntax (expected end of string, found end of JSON).");
} }
StringRef JSON::getRawName() const std::string_view JSON::getRawName() const
{ {
return getRawString(); return getRawString();
} }

View File

@ -136,8 +136,8 @@ public:
std::string getName() const; /// Получить имя name-value пары. std::string getName() const; /// Получить имя name-value пары.
JSON getValue() const; /// Получить значение name-value пары. JSON getValue() const; /// Получить значение name-value пары.
StringRef getRawString() const; std::string_view getRawString() const;
StringRef getRawName() const; std::string_view getRawName() const;
/// Получить значение элемента; если элемент - строка, то распарсить значение из строки; если не строка или число - то исключение. /// Получить значение элемента; если элемент - строка, то распарсить значение из строки; если не строка или число - то исключение.
double toDouble() const; double toDouble() const;

View File

@ -10,6 +10,14 @@
#define ARRAY_SIZE(a) sizeof((a))/sizeof((a[0])) #define ARRAY_SIZE(a) sizeof((a))/sizeof((a[0]))
/// Suppress TSan since it is possible for this code to be called from multiple threads,
/// and initialization is safe to be done multiple times from multiple threads.
#if defined(__clang__)
# define NO_SANITIZE_THREAD __attribute__((__no_sanitize__("thread")))
#else
# define NO_SANITIZE_THREAD
#endif
// We don't have libc struct available here. // We don't have libc struct available here.
// Compute aux vector manually (from /proc/self/auxv). // Compute aux vector manually (from /proc/self/auxv).
// //
@ -44,7 +52,7 @@ ssize_t __retry_read(int fd, void * buf, size_t count)
return ret; return ret;
} }
} }
unsigned long __getauxval_procfs(unsigned long type) unsigned long NO_SANITIZE_THREAD __getauxval_procfs(unsigned long type)
{ {
if (type == AT_SECURE) if (type == AT_SECURE)
{ {
@ -59,7 +67,7 @@ unsigned long __getauxval_procfs(unsigned long type)
return __auxv_procfs[type]; return __auxv_procfs[type];
} }
static unsigned long __auxv_init_procfs(unsigned long type) static unsigned long NO_SANITIZE_THREAD __auxv_init_procfs(unsigned long type)
{ {
// For debugging: // For debugging:
// - od -t dL /proc/self/auxv // - od -t dL /proc/self/auxv
@ -82,6 +90,15 @@ static unsigned long __auxv_init_procfs(unsigned long type)
_Static_assert(sizeof(aux) < 4096, "Unexpected sizeof(aux)"); _Static_assert(sizeof(aux) < 4096, "Unexpected sizeof(aux)");
while (__retry_read(fd, &aux, sizeof(aux)) == sizeof(aux)) while (__retry_read(fd, &aux, sizeof(aux)) == sizeof(aux))
{ {
if (aux.a_type == AT_NULL)
{
break;
}
if (aux.a_type == AT_IGNORE || aux.a_type == AT_IGNOREPPC)
{
continue;
}
if (aux.a_type >= ARRAY_SIZE(__auxv_procfs)) if (aux.a_type >= ARRAY_SIZE(__auxv_procfs))
{ {
fprintf(stderr, "AT_* is out of range: %li (maximum allowed is %zu)\n", aux.a_type, ARRAY_SIZE(__auxv_procfs)); fprintf(stderr, "AT_* is out of range: %li (maximum allowed is %zu)\n", aux.a_type, ARRAY_SIZE(__auxv_procfs));
@ -89,8 +106,7 @@ static unsigned long __auxv_init_procfs(unsigned long type)
} }
if (__auxv_procfs[aux.a_type]) if (__auxv_procfs[aux.a_type])
{ {
fprintf(stderr, "AUXV already has value (%zu)\n", __auxv_procfs[aux.a_type]); /// It is possible due to race on initialization.
abort();
} }
__auxv_procfs[aux.a_type] = aux.a_un.a_val; __auxv_procfs[aux.a_type] = aux.a_un.a_val;
} }
@ -114,7 +130,7 @@ static unsigned long __auxv_init_procfs(unsigned long type)
// LSan will not work with __auxv_init_environ(), // LSan will not work with __auxv_init_environ(),
// since it needs getauxval() before. // since it needs getauxval() before.
// //
static size_t __find_auxv(unsigned long type) static size_t NO_SANITIZE_THREAD __find_auxv(unsigned long type)
{ {
size_t i; size_t i;
for (i = 0; __auxv_environ[i]; i += 2) for (i = 0; __auxv_environ[i]; i += 2)
@ -126,7 +142,7 @@ static size_t __find_auxv(unsigned long type)
} }
return (size_t) -1; return (size_t) -1;
} }
unsigned long __getauxval_environ(unsigned long type) unsigned long NO_SANITIZE_THREAD __getauxval_environ(unsigned long type)
{ {
if (type == AT_SECURE) if (type == AT_SECURE)
return __auxv_secure; return __auxv_secure;
@ -141,7 +157,7 @@ unsigned long __getauxval_environ(unsigned long type)
errno = ENOENT; errno = ENOENT;
return 0; return 0;
} }
static unsigned long __auxv_init_environ(unsigned long type) static unsigned long NO_SANITIZE_THREAD __auxv_init_environ(unsigned long type)
{ {
if (!__environ) if (!__environ)
{ {

View File

@ -2,11 +2,11 @@
# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
SET(VERSION_REVISION 54464) SET(VERSION_REVISION 54465)
SET(VERSION_MAJOR 22) SET(VERSION_MAJOR 22)
SET(VERSION_MINOR 7) SET(VERSION_MINOR 8)
SET(VERSION_PATCH 1) SET(VERSION_PATCH 1)
SET(VERSION_GITHASH 7000c4e0033bb9e69050ab8ef73e8e7465f78059) SET(VERSION_GITHASH f4f05ec786a8b8966dd0ea2a2d7e39a8c7db24f4)
SET(VERSION_DESCRIBE v22.7.1.1-testing) SET(VERSION_DESCRIBE v22.8.1.1-testing)
SET(VERSION_STRING 22.7.1.1) SET(VERSION_STRING 22.8.1.1)
# end of autochange # end of autochange

View File

@ -1,3 +1,19 @@
if (_CLICKHOUSE_TOOLCHAIN_FILE_LOADED)
# During first run of cmake the toolchain file will be loaded twice,
# - /usr/share/cmake-3.23/Modules/CMakeDetermineSystem.cmake
# - /bld/CMakeFiles/3.23.2/CMakeSystem.cmake
#
# But once you already have non-empty cmake cache it will be loaded only
# once:
# - /bld/CMakeFiles/3.23.2/CMakeSystem.cmake
#
# This has no harm except for double load of toolchain will add
# --gcc-toolchain multiple times that will not allow ccache to reuse the
# cache.
return()
endif()
set (_CLICKHOUSE_TOOLCHAIN_FILE_LOADED ON)
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
set (CMAKE_SYSTEM_NAME "Linux") set (CMAKE_SYSTEM_NAME "Linux")

View File

@ -156,7 +156,8 @@ endif()
add_contrib (sqlite-cmake sqlite-amalgamation) add_contrib (sqlite-cmake sqlite-amalgamation)
add_contrib (s2geometry-cmake s2geometry) add_contrib (s2geometry-cmake s2geometry)
add_contrib (base-x-cmake base-x) add_contrib(c-ares-cmake c-ares)
add_contrib (qpl-cmake qpl)
# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs. # Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear

1
contrib/base-x vendored

@ -1 +0,0 @@
Subproject commit a85f98fb4ed52c2f4029a4b6ac1ef0bafdfc56f5

View File

@ -1,28 +0,0 @@
option (ENABLE_BASEX "Enable base-x" ${ENABLE_LIBRARIES})
if (NOT ENABLE_BASEX)
message(STATUS "Not using base-x")
return()
endif()
set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/base-x")
set (SRCS
${LIBRARY_DIR}/base_x.hh
${LIBRARY_DIR}/uinteger_t.hh
)
add_library(_base-x INTERFACE)
target_include_directories(_base-x SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/base-x")
if (XCODE OR XCODE_VERSION)
# https://gitlab.kitware.com/cmake/cmake/issues/17457
# Some native build systems may not like targets that have only object files, so consider adding at least one real source file
# This applies to Xcode.
if (NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/dummy.c")
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/dummy.c" "")
endif ()
target_sources(_base-x PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/dummy.c")
endif ()
add_library(ch_contrib::base-x ALIAS _base-x)

1
contrib/c-ares vendored Submodule

@ -0,0 +1 @@
Subproject commit afee6748b0b99acf4509d42fa37ac8422262f91b

View File

@ -0,0 +1,35 @@
# Choose to build static or shared library for c-ares.
if (USE_STATIC_LIBRARIES)
set(CARES_STATIC ON CACHE BOOL "" FORCE)
set(CARES_SHARED OFF CACHE BOOL "" FORCE)
else ()
set(CARES_STATIC OFF CACHE BOOL "" FORCE)
set(CARES_SHARED ON CACHE BOOL "" FORCE)
endif ()
# Disable looking for libnsl on a platforms that has gethostbyname in glibc
#
# c-ares searching for gethostbyname in the libnsl library, however in the
# version that shipped with gRPC it doing it wrong [1], since it uses
# CHECK_LIBRARY_EXISTS(), which will return TRUE even if the function exists in
# another dependent library. The upstream already contains correct macro [2],
# but it is not included in gRPC (even upstream gRPC, not the one that is
# shipped with clickhousee).
#
# [1]: https://github.com/c-ares/c-ares/blob/e982924acee7f7313b4baa4ee5ec000c5e373c30/CMakeLists.txt#L125
# [2]: https://github.com/c-ares/c-ares/blob/44fbc813685a1fa8aa3f27fcd7544faf612d376a/CMakeLists.txt#L146
#
# And because if you by some reason have libnsl [3] installed, clickhouse will
# reject to start w/o it. While this is completelly different library.
#
# [3]: https://packages.debian.org/bullseye/libnsl2
if (NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS")
set(HAVE_LIBNSL OFF CACHE BOOL "" FORCE)
endif()
# Force use of c-ares inet_net_pton instead of libresolv one
set(HAVE_INET_NET_PTON OFF CACHE BOOL "" FORCE)
add_subdirectory("../c-ares/" "../c-ares/")
add_library(ch_contrib::c-ares ALIAS c-ares)

View File

@ -45,38 +45,11 @@ set(_gRPC_SSL_LIBRARIES OpenSSL::Crypto OpenSSL::SSL)
# Use abseil-cpp from ClickHouse contrib, not from gRPC third_party. # Use abseil-cpp from ClickHouse contrib, not from gRPC third_party.
set(gRPC_ABSL_PROVIDER "clickhouse" CACHE STRING "" FORCE) set(gRPC_ABSL_PROVIDER "clickhouse" CACHE STRING "" FORCE)
# Choose to build static or shared library for c-ares.
if (USE_STATIC_LIBRARIES)
set(CARES_STATIC ON CACHE BOOL "" FORCE)
set(CARES_SHARED OFF CACHE BOOL "" FORCE)
else ()
set(CARES_STATIC OFF CACHE BOOL "" FORCE)
set(CARES_SHARED ON CACHE BOOL "" FORCE)
endif ()
# Disable looking for libnsl on a platforms that has gethostbyname in glibc
#
# c-ares searching for gethostbyname in the libnsl library, however in the
# version that shipped with gRPC it doing it wrong [1], since it uses
# CHECK_LIBRARY_EXISTS(), which will return TRUE even if the function exists in
# another dependent library. The upstream already contains correct macro [2],
# but it is not included in gRPC (even upstream gRPC, not the one that is
# shipped with clickhousee).
#
# [1]: https://github.com/c-ares/c-ares/blob/e982924acee7f7313b4baa4ee5ec000c5e373c30/CMakeLists.txt#L125
# [2]: https://github.com/c-ares/c-ares/blob/44fbc813685a1fa8aa3f27fcd7544faf612d376a/CMakeLists.txt#L146
#
# And because if you by some reason have libnsl [3] installed, clickhouse will
# reject to start w/o it. While this is completelly different library.
#
# [3]: https://packages.debian.org/bullseye/libnsl2
if (NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS")
set(HAVE_LIBNSL OFF CACHE BOOL "" FORCE)
endif()
# We don't want to build C# extensions. # We don't want to build C# extensions.
set(gRPC_BUILD_CSHARP_EXT OFF) set(gRPC_BUILD_CSHARP_EXT OFF)
set(_gRPC_CARES_LIBRARIES ch_contrib::c-ares)
set(gRPC_CARES_PROVIDER "clickhouse" CACHE STRING "" FORCE)
add_subdirectory("${_gRPC_SOURCE_DIR}" "${_gRPC_BINARY_DIR}") add_subdirectory("${_gRPC_SOURCE_DIR}" "${_gRPC_BINARY_DIR}")
# The contrib/grpc/CMakeLists.txt redefined the PROTOBUF_GENERATE_GRPC_CPP() function for its own purposes, # The contrib/grpc/CMakeLists.txt redefined the PROTOBUF_GENERATE_GRPC_CPP() function for its own purposes,

View File

@ -93,6 +93,18 @@ set (CMAKE_CXX_STANDARD 17)
set (LLVM_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm") set (LLVM_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm")
set (LLVM_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm") set (LLVM_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm")
add_subdirectory ("${LLVM_SOURCE_DIR}" "${LLVM_BINARY_DIR}") add_subdirectory ("${LLVM_SOURCE_DIR}" "${LLVM_BINARY_DIR}")
set_directory_properties (PROPERTIES
# due to llvm crosscompile cmake does not know how to clean it, and on clean
# will lead to the following error:
#
# ninja: error: remove(contrib/llvm/llvm/NATIVE): Directory not empty
#
ADDITIONAL_CLEAN_FILES "${LLVM_BINARY_DIR}"
# llvm's cmake configuring this file only when cmake runs,
# and after clean cmake will not know that it should re-run,
# add explicitly depends from llvm-config.h
CMAKE_CONFIGURE_DEPENDS "${LLVM_BINARY_DIR}/include/llvm/Config/llvm-config.h"
)
add_library (_llvm INTERFACE) add_library (_llvm INTERFACE)
target_link_libraries (_llvm INTERFACE ${REQUIRED_LLVM_LIBRARIES}) target_link_libraries (_llvm INTERFACE ${REQUIRED_LLVM_LIBRARIES})

1
contrib/qpl vendored Submodule

@ -0,0 +1 @@
Subproject commit cdc8442f7a5e7a6ff6eea39c69665e0c5034d85d

View File

@ -0,0 +1,322 @@
## The Intel® QPL provides high performance implementations of data processing functions for existing hardware accelerator, and/or software path in case if hardware accelerator is not available.
if (OS_LINUX AND ARCH_AMD64 AND (ENABLE_AVX2 OR ENABLE_AVX512))
option (ENABLE_QPL "Enable Intel® Query Processing Library" ${ENABLE_LIBRARIES})
elseif(ENABLE_QPL)
message (${RECONFIGURE_MESSAGE_LEVEL} "QPL library is only supported on x86_64 arch with avx2/avx512 support")
endif()
if (NOT ENABLE_QPL)
message(STATUS "Not using QPL")
return()
endif()
set (QPL_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl")
set (QPL_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl/sources")
set (QPL_BINARY_DIR "${ClickHouse_BINARY_DIR}/build/contrib/qpl")
set (UUID_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake")
set (EFFICIENT_WAIT ON)
set (BLOCK_ON_FAULT ON)
set (LOG_HW_INIT OFF)
set (SANITIZE_MEMORY OFF)
set (SANITIZE_THREADS OFF)
set (LIB_FUZZING_ENGINE OFF)
function(GetLibraryVersion _content _outputVar)
string(REGEX MATCHALL "Qpl VERSION (.+) LANGUAGES" VERSION_REGEX "${_content}")
SET(${_outputVar} ${CMAKE_MATCH_1} PARENT_SCOPE)
endfunction()
FILE(READ "${QPL_PROJECT_DIR}/CMakeLists.txt" HEADER_CONTENT)
GetLibraryVersion("${HEADER_CONTENT}" QPL_VERSION)
message(STATUS "Intel QPL version: ${QPL_VERSION}")
# There are 5 source subdirectories under $QPL_SRC_DIR: isal, c_api, core-sw, middle-layer, c_api.
# Generate 7 library targets: middle_layer_lib, isal, isal_asm, qplcore_px, qplcore_avx512, core_iaa, middle_layer_lib.
# Output ch_contrib::qpl by linking with 7 library targets.
include("${QPL_PROJECT_DIR}/cmake/CompileOptions.cmake")
# check nasm compiler
include(CheckLanguage)
check_language(ASM_NASM)
if(NOT CMAKE_ASM_NASM_COMPILER)
message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!")
endif()
# [SUBDIR]isal
enable_language(ASM_NASM)
set(ISAL_C_SRC ${QPL_SRC_DIR}/isal/igzip/adler32_base.c
${QPL_SRC_DIR}/isal/igzip/huff_codes.c
${QPL_SRC_DIR}/isal/igzip/hufftables_c.c
${QPL_SRC_DIR}/isal/igzip/igzip.c
${QPL_SRC_DIR}/isal/igzip/igzip_base.c
${QPL_SRC_DIR}/isal/igzip/flatten_ll.c
${QPL_SRC_DIR}/isal/igzip/encode_df.c
${QPL_SRC_DIR}/isal/igzip/igzip_icf_base.c
${QPL_SRC_DIR}/isal/igzip/igzip_inflate.c
${QPL_SRC_DIR}/isal/igzip/igzip_icf_body.c
${QPL_SRC_DIR}/isal/crc/crc_base.c
${QPL_SRC_DIR}/isal/crc/crc64_base.c)
set(ISAL_ASM_SRC ${QPL_SRC_DIR}/isal/igzip/igzip_body.asm
${QPL_SRC_DIR}/isal/igzip/igzip_gen_icf_map_lh1_04.asm
${QPL_SRC_DIR}/isal/igzip/igzip_gen_icf_map_lh1_06.asm
${QPL_SRC_DIR}/isal/igzip/igzip_decode_block_stateless_04.asm
${QPL_SRC_DIR}/isal/igzip/igzip_finish.asm
${QPL_SRC_DIR}/isal/igzip/encode_df_04.asm
${QPL_SRC_DIR}/isal/igzip/encode_df_06.asm
${QPL_SRC_DIR}/isal/igzip/igzip_decode_block_stateless_01.asm
${QPL_SRC_DIR}/isal/igzip/proc_heap.asm
${QPL_SRC_DIR}/isal/igzip/igzip_icf_body_h1_gr_bt.asm
${QPL_SRC_DIR}/isal/igzip/igzip_icf_finish.asm
${QPL_SRC_DIR}/isal/igzip/igzip_inflate_multibinary.asm
${QPL_SRC_DIR}/isal/igzip/igzip_update_histogram_01.asm
${QPL_SRC_DIR}/isal/igzip/igzip_update_histogram_04.asm
${QPL_SRC_DIR}/isal/igzip/rfc1951_lookup.asm
${QPL_SRC_DIR}/isal/igzip/adler32_sse.asm
${QPL_SRC_DIR}/isal/igzip/adler32_avx2_4.asm
${QPL_SRC_DIR}/isal/igzip/igzip_deflate_hash.asm
${QPL_SRC_DIR}/isal/igzip/igzip_set_long_icf_fg_04.asm
${QPL_SRC_DIR}/isal/igzip/igzip_set_long_icf_fg_06.asm
${QPL_SRC_DIR}/isal/igzip/igzip_multibinary.asm
${QPL_SRC_DIR}/isal/igzip/stdmac.asm
${QPL_SRC_DIR}/isal/crc/crc_multibinary.asm
${QPL_SRC_DIR}/isal/crc/crc32_gzip_refl_by8.asm
${QPL_SRC_DIR}/isal/crc/crc32_gzip_refl_by8_02.asm
${QPL_SRC_DIR}/isal/crc/crc32_gzip_refl_by16_10.asm
${QPL_SRC_DIR}/isal/crc/crc32_ieee_01.asm
${QPL_SRC_DIR}/isal/crc/crc32_ieee_02.asm
${QPL_SRC_DIR}/isal/crc/crc32_ieee_by4.asm
${QPL_SRC_DIR}/isal/crc/crc32_ieee_by16_10.asm
${QPL_SRC_DIR}/isal/crc/crc32_iscsi_00.asm
${QPL_SRC_DIR}/isal/crc/crc32_iscsi_01.asm
${QPL_SRC_DIR}/isal/crc/crc32_iscsi_by16_10.asm)
# Adding ISA-L library target
add_library(isal OBJECT ${ISAL_C_SRC})
add_library(isal_asm OBJECT ${ISAL_ASM_SRC})
# Setting external and internal interfaces for ISA-L library
target_include_directories(isal
PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/isal/include>
PRIVATE ${QPL_SRC_DIR}/isal/include
PUBLIC ${QPL_SRC_DIR}/isal/igzip)
target_compile_options(isal PRIVATE
"$<$<C_COMPILER_ID:GNU>:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}>"
"$<$<CONFIG:Debug>:>"
"$<$<CONFIG:Release>:>")
target_compile_options(isal_asm PUBLIC "-I${QPL_SRC_DIR}/isal/include/"
PUBLIC "-I${QPL_SRC_DIR}/isal/igzip/"
PUBLIC "-I${QPL_SRC_DIR}/isal/crc/"
PUBLIC "-DQPL_LIB")
# AS_FEATURE_LEVEL=10 means "Check SIMD capabilities of the target system at runtime and use up to AVX512 if available".
# AS_FEATURE_LEVEL=5 means "Check SIMD capabilities of the target system at runtime and use up to AVX2 if available".
# HAVE_KNOWS_AVX512 means rely on AVX512 being available on the target system.
if (ENABLE_AVX512)
target_compile_options(isal_asm PUBLIC "-DHAVE_AS_KNOWS_AVX512" "-DAS_FEATURE_LEVEL=10")
else()
target_compile_options(isal_asm PUBLIC "-DAS_FEATURE_LEVEL=5")
endif()
# Here must remove "-fno-sanitize=undefined" from COMPILE_OPTIONS.
# Otherwise nasm compiler would fail to proceed due to unrecognition of "-fno-sanitize=undefined"
if (SANITIZE STREQUAL "undefined")
get_target_property(target_options isal_asm COMPILE_OPTIONS)
list(REMOVE_ITEM target_options "-fno-sanitize=undefined")
set_property(TARGET isal_asm PROPERTY COMPILE_OPTIONS ${target_options})
endif()
target_compile_definitions(isal PUBLIC
QPL_LIB
NDEBUG)
# [SUBDIR]core-sw
# Two libraries:qplcore_avx512/qplcore_px for SW fallback will be created which are implemented by AVX512 and non-AVX512 instructions respectively.
# The upper level QPL API will check SIMD capabilities of the target system at runtime and decide to call AVX512 function or non-AVX512 function.
# Hence, here we don't need put qplcore_avx512 under an ENABLE_AVX512 CMake switch.
# Actually, if we do that, some undefined symbols errors would happen because both of AVX512 function and non-AVX512 function are referenced by QPL API.
# PLATFORM=2 means AVX512 implementation; PLATFORM=0 means non-AVX512 implementation.
# Find Core Sources
file(GLOB SOURCES
${QPL_SRC_DIR}/core-sw/src/checksums/*.c
${QPL_SRC_DIR}/core-sw/src/filtering/*.c
${QPL_SRC_DIR}/core-sw/src/other/*.c
${QPL_SRC_DIR}/core-sw/src/compression/*.c)
file(GLOB DATA_SOURCES
${QPL_SRC_DIR}/core-sw/src/data/*.c)
# Create avx512 library
add_library(qplcore_avx512 OBJECT ${SOURCES})
target_compile_definitions(qplcore_avx512 PRIVATE PLATFORM=2)
target_include_directories(qplcore_avx512
PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/include>
PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/src/include>
PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/src/compression/include>
PRIVATE $<TARGET_PROPERTY:isal,INTERFACE_INCLUDE_DIRECTORIES>)
set_target_properties(qplcore_avx512 PROPERTIES
$<$<C_COMPILER_ID:GNU>:C_STANDARD 17>)
target_link_libraries(qplcore_avx512 ${CMAKE_DL_LIBS} isal)
target_compile_options(qplcore_avx512
PRIVATE ${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}
PRIVATE -march=skylake-avx512
PRIVATE "$<$<CONFIG:Debug>:>"
PRIVATE "$<$<CONFIG:Release>:-O3;-D_FORTIFY_SOURCE=2>")
target_compile_definitions(qplcore_avx512 PUBLIC QPL_BADARG_CHECK)
#
# Create px library
#
#set(CMAKE_INCLUDE_CURRENT_DIR ON)
# Create library
add_library(qplcore_px OBJECT ${SOURCES} ${DATA_SOURCES})
target_compile_definitions(qplcore_px PRIVATE PLATFORM=0)
target_include_directories(qplcore_px
PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/include>
PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/src/include>
PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-sw/src/compression/include>
PRIVATE $<TARGET_PROPERTY:isal,INTERFACE_INCLUDE_DIRECTORIES>)
set_target_properties(qplcore_px PROPERTIES
$<$<C_COMPILER_ID:GNU>:C_STANDARD 17>)
target_link_libraries(qplcore_px isal ${CMAKE_DL_LIBS})
target_compile_options(qplcore_px
PRIVATE ${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}
PRIVATE "$<$<CONFIG:Debug>:>"
PRIVATE "$<$<CONFIG:Release>:-O3;-D_FORTIFY_SOURCE=2>")
target_compile_definitions(qplcore_px PUBLIC QPL_BADARG_CHECK)
# [SUBDIR]core-iaa
file(GLOB HW_PATH_SRC ${QPL_SRC_DIR}/core-iaa/sources/aecs/*.c
${QPL_SRC_DIR}/core-iaa/sources/aecs/*.cpp
${QPL_SRC_DIR}/core-iaa/sources/driver_loader/*.c
${QPL_SRC_DIR}/core-iaa/sources/driver_loader/*.cpp
${QPL_SRC_DIR}/core-iaa/sources/descriptors/*.c
${QPL_SRC_DIR}/core-iaa/sources/descriptors/*.cpp
${QPL_SRC_DIR}/core-iaa/sources/bit_rev.c)
# Create library
add_library(core_iaa OBJECT ${HW_PATH_SRC})
target_include_directories(core_iaa
PRIVATE ${UUID_DIR}
PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-iaa/include>
PRIVATE $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-iaa/sources/include>
PRIVATE $<TARGET_PROPERTY:qplcore_avx512,INTERFACE_INCLUDE_DIRECTORIES>)
target_compile_options(core_iaa
PRIVATE $<$<C_COMPILER_ID:GNU>:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS};
$<$<CONFIG:Release>:-O3;-D_FORTIFY_SOURCE=2>>)
target_compile_features(core_iaa PRIVATE c_std_11)
target_compile_definitions(core_iaa PRIVATE QPL_BADARG_CHECK
PRIVATE $<$<BOOL:${BLOCK_ON_FAULT}>: BLOCK_ON_FAULT_ENABLED>
PRIVATE $<$<BOOL:${LOG_HW_INIT}>:LOG_HW_INIT>)
# [SUBDIR]middle-layer
generate_unpack_kernel_arrays(${QPL_BINARY_DIR})
file(GLOB MIDDLE_LAYER_SRC
${QPL_SRC_DIR}/middle-layer/analytics/*.cpp
${QPL_SRC_DIR}/middle-layer/c_wrapper/*.cpp
${QPL_SRC_DIR}/middle-layer/checksum/*.cpp
${QPL_SRC_DIR}/middle-layer/common/*.cpp
${QPL_SRC_DIR}/middle-layer/compression/*.cpp
${QPL_SRC_DIR}/middle-layer/compression/*/*.cpp
${QPL_SRC_DIR}/middle-layer/compression/*/*/*.cpp
${QPL_SRC_DIR}/middle-layer/dispatcher/*.cpp
${QPL_SRC_DIR}/middle-layer/other/*.cpp
${QPL_SRC_DIR}/middle-layer/util/*.cpp
${QPL_SRC_DIR}/middle-layer/inflate/*.cpp
${QPL_SRC_DIR}/core-iaa/sources/accelerator/*.cpp) # todo
file(GLOB GENERATED_PX_TABLES_SRC ${QPL_BINARY_DIR}/generated/px_*.cpp)
file(GLOB GENERATED_AVX512_TABLES_SRC ${QPL_BINARY_DIR}/generated/avx512_*.cpp)
add_library(middle_layer_lib OBJECT
${GENERATED_PX_TABLES_SRC}
${GENERATED_AVX512_TABLES_SRC}
${MIDDLE_LAYER_SRC})
target_compile_options(middle_layer_lib
PRIVATE $<$<C_COMPILER_ID:GNU>:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS};
${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS};
$<$<CONFIG:Release>:-O3;-D_FORTIFY_SOURCE=2>>
PRIVATE $<$<COMPILE_LANG_AND_ID:CXX,GNU>:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>)
target_compile_definitions(middle_layer_lib
PUBLIC QPL_VERSION="${QPL_VERSION}"
PUBLIC $<$<BOOL:${LOG_HW_INIT}>:LOG_HW_INIT>
PUBLIC $<$<BOOL:${EFFICIENT_WAIT}>:QPL_EFFICIENT_WAIT>
PUBLIC QPL_BADARG_CHECK)
set_source_files_properties(${GENERATED_PX_TABLES_SRC} PROPERTIES COMPILE_DEFINITIONS PLATFORM=0)
set_source_files_properties(${GENERATED_AVX512_TABLES_SRC} PROPERTIES COMPILE_DEFINITIONS PLATFORM=2)
target_include_directories(middle_layer_lib
PRIVATE ${UUID_DIR}
PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/middle-layer>
PUBLIC $<TARGET_PROPERTY:_qpl,INTERFACE_INCLUDE_DIRECTORIES>
PUBLIC $<TARGET_PROPERTY:qplcore_px,INTERFACE_INCLUDE_DIRECTORIES>
PUBLIC $<TARGET_PROPERTY:qplcore_avx512,INTERFACE_INCLUDE_DIRECTORIES>
PUBLIC $<TARGET_PROPERTY:isal,INTERFACE_INCLUDE_DIRECTORIES>
PUBLIC $<TARGET_PROPERTY:core_iaa,INTERFACE_INCLUDE_DIRECTORIES>)
target_compile_definitions(middle_layer_lib PUBLIC -DQPL_LIB)
# [SUBDIR]c_api
file(GLOB_RECURSE QPL_C_API_SRC
${QPL_SRC_DIR}/c_api/*.c
${QPL_SRC_DIR}/c_api/*.cpp)
add_library(_qpl STATIC ${QPL_C_API_SRC}
$<TARGET_OBJECTS:middle_layer_lib>
$<TARGET_OBJECTS:isal>
$<TARGET_OBJECTS:isal_asm>
$<TARGET_OBJECTS:qplcore_px>
$<TARGET_OBJECTS:qplcore_avx512>
$<TARGET_OBJECTS:core_iaa>
$<TARGET_OBJECTS:middle_layer_lib>)
target_include_directories(_qpl
PUBLIC $<BUILD_INTERFACE:${QPL_PROJECT_DIR}/include/>
PRIVATE $<TARGET_PROPERTY:middle_layer_lib,INTERFACE_INCLUDE_DIRECTORIES>
PRIVATE $<BUILD_INTERFACE:${QPL_SRC_DIR}/c_api>)
target_compile_options(_qpl
PRIVATE $<$<C_COMPILER_ID:GNU>:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS};
${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS};
$<$<CONFIG:Release>:-O3;-D_FORTIFY_SOURCE=2>>
PRIVATE $<$<COMPILE_LANG_AND_ID:CXX,GNU>:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>)
target_compile_definitions(_qpl
PRIVATE -DQPL_LIB
PRIVATE -DQPL_BADARG_CHECK
PUBLIC -DENABLE_QPL_COMPRESSION)
target_link_libraries(_qpl
PRIVATE ${CMAKE_DL_LIBS})
add_library (ch_contrib::qpl ALIAS _qpl)
target_include_directories(_qpl SYSTEM BEFORE PUBLIC "${QPL_PROJECT_DIR}/include")

View File

@ -0,0 +1,4 @@
#ifndef _QPL_UUID_UUID_H
#define _QPL_UUID_UUID_H
typedef unsigned char uuid_t[16];
#endif /* _QPL_UUID_UUID_H */

View File

@ -51,6 +51,7 @@ RUN apt-get update \
rename \ rename \
software-properties-common \ software-properties-common \
tzdata \ tzdata \
nasm \
--yes --no-install-recommends \ --yes --no-install-recommends \
&& apt-get clean && apt-get clean

View File

@ -55,6 +55,7 @@ RUN apt-get update \
pkg-config \ pkg-config \
tzdata \ tzdata \
pv \ pv \
nasm \
--yes --no-install-recommends --yes --no-install-recommends
# Sanitizer options for services (clickhouse-server) # Sanitizer options for services (clickhouse-server)

View File

@ -72,6 +72,7 @@ RUN apt-get update \
tzdata \ tzdata \
unixodbc \ unixodbc \
file \ file \
nasm \
--yes --no-install-recommends --yes --no-install-recommends
RUN pip3 install numpy scipy pandas Jinja2 RUN pip3 install numpy scipy pandas Jinja2

View File

@ -135,6 +135,7 @@ function clone_submodules
contrib/replxx contrib/replxx
contrib/wyhash contrib/wyhash
contrib/hashidsxx contrib/hashidsxx
contrib/c-ares
) )
git submodule sync git submodule sync

View File

@ -0,0 +1,9 @@
version: "2.3"
services:
coredns:
image: coredns/coredns:latest
restart: always
volumes:
- ${COREDNS_CONFIG_DIR}/example.com:/example.com
- ${COREDNS_CONFIG_DIR}/Corefile:/Corefile

View File

@ -17,7 +17,9 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
python3-pip \ python3-pip \
shellcheck \ shellcheck \
yamllint \ yamllint \
&& pip3 install black boto3 codespell dohq-artifactory PyGithub unidiff pylint==2.6.2 && pip3 install black boto3 codespell dohq-artifactory PyGithub unidiff pylint==2.6.2 \
&& apt-get clean \
&& rm -rf /root/.cache/pip
# Architecture of the image when BuildKit/buildx is used # Architecture of the image when BuildKit/buildx is used
ARG TARGETARCH ARG TARGETARCH

View File

@ -40,10 +40,10 @@ def process_result(result_folder):
def write_results(results_file, status_file, results, status): def write_results(results_file, status_file, results, status):
with open(results_file, "w") as f: with open(results_file, "w", encoding="utf-8") as f:
out = csv.writer(f, delimiter="\t") out = csv.writer(f, delimiter="\t")
out.writerows(results) out.writerows(results)
with open(status_file, "w") as f: with open(status_file, "w", encoding="utf-8") as f:
out = csv.writer(f, delimiter="\t") out = csv.writer(f, delimiter="\t")
out.writerow(status) out.writerow(status)
@ -53,9 +53,10 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="ClickHouse script for parsing results of style check" description="ClickHouse script for parsing results of style check"
) )
parser.add_argument("--in-results-dir", default="/test_output/") default_dir = "/test_output"
parser.add_argument("--out-results-file", default="/test_output/test_results.tsv") parser.add_argument("--in-results-dir", default=default_dir)
parser.add_argument("--out-status-file", default="/test_output/check_status.tsv") parser.add_argument("--out-results-file", default=f"{default_dir}/test_results.tsv")
parser.add_argument("--out-status-file", default=f"{default_dir}/check_status.tsv")
args = parser.parse_args() args = parser.parse_args()
state, description, test_results = process_result(args.in_results_dir) state, description, test_results = process_result(args.in_results_dir)

View File

@ -38,9 +38,9 @@ Writing the docs is extremely useful for project's users and developers, and gro
The documentation contains information about all the aspects of the ClickHouse lifecycle: developing, testing, installing, operating, and using. The base language of the documentation is English. The English version is the most actual. All other languages are supported as much as they can by contributors from different countries. The documentation contains information about all the aspects of the ClickHouse lifecycle: developing, testing, installing, operating, and using. The base language of the documentation is English. The English version is the most actual. All other languages are supported as much as they can by contributors from different countries.
At the moment, [documentation](https://clickhouse.com/docs) exists in English, Russian, Chinese, Japanese. We store the documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs). At the moment, [documentation](https://clickhouse.com/docs) exists in English, Russian, and Chinese. We store the reference documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs), and user guides in a separate repo [Clickhouse/clickhouse-docs](https://github.com/ClickHouse/clickhouse-docs).
Each language lays in the corresponding folder. Files that are not translated from English are the symbolic links to the English ones. Each language lies in the corresponding folder. Files that are not translated from English are symbolic links to the English ones.
<a name="how-to-contribute"/> <a name="how-to-contribute"/>
@ -48,9 +48,9 @@ Each language lays in the corresponding folder. Files that are not translated fr
You can contribute to the documentation in many ways, for example: You can contribute to the documentation in many ways, for example:
- Fork the ClickHouse repository, edit, commit, push, and open a pull request. - Fork the ClickHouse and ClickHouse-docs repositories, edit, commit, push, and open a pull request.
Add the `documentation` label to this pull request for proper automatic checks applying. If you have no permissions for adding labels, the reviewer of your PR adds it. Add the `pr-documentation` label to this pull request for proper automatic checks applying. If you do not have permission to add labels, then the reviewer of your PR will add it.
- Open a required file in the ClickHouse repository and edit it from the GitHub web interface. - Open a required file in the ClickHouse repository and edit it from the GitHub web interface.
@ -158,15 +158,15 @@ When everything is ready, we will add the new language to the website.
<a name="target-audience"/> <a name="target-audience"/>
### Documentation for Different Audience ### Documentation for Different Audiences
When writing documentation, think about people who read it. Each audience has specific requirements for terms they use in communications. When writing documentation, think about the people who read it. Each audience has specific requirements for terms they use in communications.
ClickHouse documentation can be divided by the audience for the following parts: ClickHouse documentation can be divided up by the audience for the following parts:
- Conceptual topics in [Introduction](https://clickhouse.com/docs/en/), tutorials and overviews, changelog. - Conceptual topics like tutorials and overviews.
These topics are for the most common auditory. When editing text in them, use the most common terms that are comfortable for the audience with basic technical skills. These topics are for the most common audience. When editing text in them, use the most common terms that are comfortable for the audience with basic technical skills.
- Query language reference and related topics. - Query language reference and related topics.

View File

@ -0,0 +1,468 @@
---
sidebar_position: 1
sidebar_label: 2022
---
# 2022 Changelog
### ClickHouse release v22.7.1.2484-stable (f4f05ec786a) FIXME as compared to v22.6.1.1985-stable (7000c4e0033)
#### Backward Incompatible Change
* Enable setting `enable_positional_arguments` by default. It allows queries like `SELECT ... ORDER BY 1, 2` where 1, 2 are the references to the select clause. If you need to return the old behavior, disable this setting. [#38204](https://github.com/ClickHouse/ClickHouse/pull/38204) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* `Ordinary` database engine and old storage definition syntax for `*MergeTree` tables are deprecated. By default it's not possible to create new ones. If `system` database has `Ordinary` engine it will be automatically converted to `Atomic` on server startup. There are settings to keep old behavior (`allow_deprecated_database_ordinary` and `allow_deprecated_syntax_for_merge_tree`), but these settings may be removed in future releases. [#38335](https://github.com/ClickHouse/ClickHouse/pull/38335) ([Alexander Tokmakov](https://github.com/tavplubix)).
* * Force rewriting comma join to inner by default (set default value `cross_to_inner_join_rewrite = 2`). To have old behavior set `cross_to_inner_join_rewrite = 1`. [#39326](https://github.com/ClickHouse/ClickHouse/pull/39326) ([Vladimir C](https://github.com/vdimir)).
* Disable format_csv_allow_single_quotes by default. [#37096](https://github.com/ClickHouse/ClickHouse/issues/37096). [#39423](https://github.com/ClickHouse/ClickHouse/pull/39423) ([Kruglov Pavel](https://github.com/Avogar)).
#### New Feature
* Add new `direct` join algorithm for RocksDB, ref [#33582](https://github.com/ClickHouse/ClickHouse/issues/33582). [#35363](https://github.com/ClickHouse/ClickHouse/pull/35363) ([Vladimir C](https://github.com/vdimir)).
* * Added full sorting merge join algorithm. [#35796](https://github.com/ClickHouse/ClickHouse/pull/35796) ([Vladimir C](https://github.com/vdimir)).
* Add a setting `zstd_window_log_max` to configure max memory usage on zstd decoding when importing external files. Closes [#35693](https://github.com/ClickHouse/ClickHouse/issues/35693). [#37015](https://github.com/ClickHouse/ClickHouse/pull/37015) ([wuxiaobai24](https://github.com/wuxiaobai24)).
* Implement NatsStorage - table engine, which allows to pub/sub to NATS. Closes [#32388](https://github.com/ClickHouse/ClickHouse/issues/32388). [#37171](https://github.com/ClickHouse/ClickHouse/pull/37171) ([tchepavel](https://github.com/tchepavel)).
* Implement table function MongoDB. Allow writes into MongoDB storage / table function. [#37213](https://github.com/ClickHouse/ClickHouse/pull/37213) ([aaapetrenko](https://github.com/aaapetrenko)).
* `clickhouse-keeper` new feature: add support for real-time digest calculation and verification. [#37555](https://github.com/ClickHouse/ClickHouse/pull/37555) ([Antonio Andelic](https://github.com/antonio2368)).
* In [#17202](https://github.com/ClickHouse/ClickHouse/issues/17202) was reported that host_regexp was being tested against only one of the possible PTR responses. This PR makes the necessary changes so that host_regexp is applied against all possible PTR responses and validate if any matches. [#37827](https://github.com/ClickHouse/ClickHouse/pull/37827) ([Arthur Passos](https://github.com/arthurpassos)).
* Support hadoop secure rpc transfer(hadoop.rpc.protection=privacy and hadoop.rpc.protection=integrity). [#37852](https://github.com/ClickHouse/ClickHouse/pull/37852) ([Peng Liu](https://github.com/michael1589)).
* Add struct type support in `StorageHive`. [#38118](https://github.com/ClickHouse/ClickHouse/pull/38118) ([lgbo](https://github.com/lgbo-ustc)).
* Added Base58 encoding/decoding. [#38159](https://github.com/ClickHouse/ClickHouse/pull/38159) ([Andrey Zvonov](https://github.com/zvonand)).
* Add chart visualization to Play UI. [#38197](https://github.com/ClickHouse/ClickHouse/pull/38197) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* support `alter` command on `StorageHive` table. [#38214](https://github.com/ClickHouse/ClickHouse/pull/38214) ([lgbo](https://github.com/lgbo-ustc)).
* Added `CREATE TABLE ... EMPTY AS SELECT` query. It automatically deduces table structure from the SELECT query, but does not fill the table after creation. Resolves [#38049](https://github.com/ClickHouse/ClickHouse/issues/38049). [#38272](https://github.com/ClickHouse/ClickHouse/pull/38272) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Adds new setting `implicit_transaction` to run standalone queries inside a transaction. It handles both creation and closing (via COMMIT if the query succeeded or ROLLBACK if it didn't) of the transaction automatically. [#38344](https://github.com/ClickHouse/ClickHouse/pull/38344) ([Raúl Marín](https://github.com/Algunenano)).
* Allow trailing comma in columns list. closes [#38425](https://github.com/ClickHouse/ClickHouse/issues/38425). [#38440](https://github.com/ClickHouse/ClickHouse/pull/38440) ([chen](https://github.com/xiedeyantu)).
* Compress clickhouse into self-extracting executable (path programs/self-extracting). New build target 'self-extracting' is added. [#38447](https://github.com/ClickHouse/ClickHouse/pull/38447) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Introduced settings `additional_table_filters`. Using this setting, you can specify additional filtering condition for a table which will be applied directly after reading. Example: `select number, x, y from (select number from system.numbers limit 5) f any left join (select x, y from table_1) s on f.number = s.x settings additional_table_filters={'system.numbers : 'number != 3', 'table_1' : 'x != 2'}`. Introduced setting `additional_result_filter` which specifies additional filtering condition for query result. Closes [#37918](https://github.com/ClickHouse/ClickHouse/issues/37918). [#38475](https://github.com/ClickHouse/ClickHouse/pull/38475) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Add SQLInsert output format. Closes [#38441](https://github.com/ClickHouse/ClickHouse/issues/38441). [#38477](https://github.com/ClickHouse/ClickHouse/pull/38477) ([Kruglov Pavel](https://github.com/Avogar)).
* Downloadable clickhouse executable is compressed self-extracting. [#38653](https://github.com/ClickHouse/ClickHouse/pull/38653) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Support `isNullable` function. This function checks whether it's argument is nullable and return true(1) or false(0). Closes [#38611](https://github.com/ClickHouse/ClickHouse/issues/38611). [#38841](https://github.com/ClickHouse/ClickHouse/pull/38841) ([lokax](https://github.com/lokax)).
* Add functions `translate(string, from_string, to_string)` and `translateUTF8(string, from_string, to_string)`. [#38935](https://github.com/ClickHouse/ClickHouse/pull/38935) ([Nikolay Degterinsky](https://github.com/evillique)).
* Add `compatibility` setting and `system.settings_changes` system table that contains information about changes in settings through ClickHouse versions. Closes [#35972](https://github.com/ClickHouse/ClickHouse/issues/35972). [#38957](https://github.com/ClickHouse/ClickHouse/pull/38957) ([Kruglov Pavel](https://github.com/Avogar)).
* Add the 3rd parameter to the tupleElement function and return it if tuple doesn't have a member. Only works if the 2nd parameter is of type String. Closes [#38872](https://github.com/ClickHouse/ClickHouse/issues/38872). [#38989](https://github.com/ClickHouse/ClickHouse/pull/38989) ([lokax](https://github.com/lokax)).
* Support parseTimedelta function. It can be used like ```sql # ' ', ';', '-', '+', ',', ':' can be used as separators, eg. "1yr-2mo", "2m:6s" SELECT parseTimeDelta('1yr-2mo-4w + 12 days, 3 hours : 1 minute ; 33 seconds');. [#39071](https://github.com/ClickHouse/ClickHouse/pull/39071) ([jiahui-97](https://github.com/jiahui-97)).
* Added options to limit IO operations with remote storage: `max_remote_read_network_bandwidth_for_server` and `max_remote_write_network_bandwidth_for_server`. [#39095](https://github.com/ClickHouse/ClickHouse/pull/39095) ([Sergei Trifonov](https://github.com/serxa)).
* Add `send_logs_source_regexp` setting. Send server text logs with specified regexp to match log source name. Empty means all sources. [#39161](https://github.com/ClickHouse/ClickHouse/pull/39161) ([Amos Bird](https://github.com/amosbird)).
* OpenTelemetry now collects traces without Processors spans by default. To enable Processors spans collection `opentelemetry_trace_processors` setting. [#39170](https://github.com/ClickHouse/ClickHouse/pull/39170) ([Ilya Yatsishin](https://github.com/qoega)).
#### Performance Improvement
* Add new `local_filesystem_read_method` method `io_uring` based on the asynchronous Linux [io_uring](https://kernel.dk/io_uring.pdf) subsystem, improving read performance almost universally compared to the default `pread` method. [#36103](https://github.com/ClickHouse/ClickHouse/pull/36103) ([Saulius Valatka](https://github.com/sauliusvl)).
* Distinct optimization for sorted columns. Use specialized distinct transformation in case input stream is sorted by column(s) in distinct. Optimization can be applied to pre-distinct, final distinct, or both. Initial implementation by @dimarub2000. [#37803](https://github.com/ClickHouse/ClickHouse/pull/37803) ([Igor Nikonov](https://github.com/devcrafter)).
* Add VBMI optimized copyOverlap32Shuffle for LZ4 decompress. [#37891](https://github.com/ClickHouse/ClickHouse/pull/37891) ([Guo Wangyang](https://github.com/guowangy)).
* Improve performance of `ORDER BY`, `MergeTree` merges, window functions using batch version of `BinaryHeap`. [#38022](https://github.com/ClickHouse/ClickHouse/pull/38022) ([Maksim Kita](https://github.com/kitaisreal)).
* Fix significant join performance regression which was introduced in https://github.com/ClickHouse/ClickHouse/pull/35616 . It's interesting that common join queries such as ssb queries have been 10 times slower for almost 3 months while no one complains. [#38052](https://github.com/ClickHouse/ClickHouse/pull/38052) ([Amos Bird](https://github.com/amosbird)).
* Migrate from the Intel hyperscan library to vectorscan, this speeds up many string matching on non-x86 platforms. [#38171](https://github.com/ClickHouse/ClickHouse/pull/38171) ([Robert Schulze](https://github.com/rschu1ze)).
* Increased parallelism of query plan steps executed after aggregation. [#38295](https://github.com/ClickHouse/ClickHouse/pull/38295) ([Nikita Taranov](https://github.com/nickitat)).
* Improve performance of insertion to columns of type `JSON`. [#38320](https://github.com/ClickHouse/ClickHouse/pull/38320) ([Anton Popov](https://github.com/CurtizJ)).
* Optimized insertion and lookups in the HashTable. [#38413](https://github.com/ClickHouse/ClickHouse/pull/38413) ([Nikita Taranov](https://github.com/nickitat)).
* Fix performance degradation from [#32493](https://github.com/ClickHouse/ClickHouse/issues/32493). [#38417](https://github.com/ClickHouse/ClickHouse/pull/38417) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Improve performance of column vector replicate using SIMD instructions. Author @zzachimed. [#38565](https://github.com/ClickHouse/ClickHouse/pull/38565) ([Maksim Kita](https://github.com/kitaisreal)).
* Norm and Distance functions for arrays speed up 1.2-2 times. [#38740](https://github.com/ClickHouse/ClickHouse/pull/38740) ([Alexander Gololobov](https://github.com/davenger)).
* A less efficient execution plan can be generated for query with ORDER BY (a, b) than for ORDER BY a, b. [#38873](https://github.com/ClickHouse/ClickHouse/pull/38873) ([Igor Nikonov](https://github.com/devcrafter)).
* Executable UDF, Executable Dictionary, Executable Storage poll subprocess fix 1 second subprocess wait during subprocess termination. [#38929](https://github.com/ClickHouse/ClickHouse/pull/38929) ([Constantine Peresypkin](https://github.com/pkit)).
* * Pushdown filter to the right side of sorting join. [#39123](https://github.com/ClickHouse/ClickHouse/pull/39123) ([Vladimir C](https://github.com/vdimir)).
* Optimize accesses to system.stack_trace. [#39177](https://github.com/ClickHouse/ClickHouse/pull/39177) ([Azat Khuzhin](https://github.com/azat)).
#### Improvement
* Optimized processing of ORDER BY in window functions. [#34632](https://github.com/ClickHouse/ClickHouse/pull/34632) ([Vladimir Chebotarev](https://github.com/excitoon)).
* Support SQL standard create index and drop index syntax. [#35166](https://github.com/ClickHouse/ClickHouse/pull/35166) ([Jianmei Zhang](https://github.com/zhangjmruc)).
* use simd to re-write the current column replicate funcion and got 2x performance boost in our unit benchmark test. [#37235](https://github.com/ClickHouse/ClickHouse/pull/37235) ([zzachimed](https://github.com/zzachimed)).
* Send profile events for INSERT queries (previously only SELECT was supported). [#37391](https://github.com/ClickHouse/ClickHouse/pull/37391) ([Azat Khuzhin](https://github.com/azat)).
* Implement in order aggregation (`optimize_aggregation_in_order`) for fully materialized projections. [#37469](https://github.com/ClickHouse/ClickHouse/pull/37469) ([Azat Khuzhin](https://github.com/azat)).
* * Bugfixes and performance improvements for `parallel_hash`. [#37648](https://github.com/ClickHouse/ClickHouse/pull/37648) ([Vladimir C](https://github.com/vdimir)).
* Support expressions with window functions. Closes [#19857](https://github.com/ClickHouse/ClickHouse/issues/19857). [#37848](https://github.com/ClickHouse/ClickHouse/pull/37848) ([Dmitry Novik](https://github.com/novikd)).
* S3 single objects are now removed with `RemoveObjectRequest` (sic). Fixed a bug with `S3ObjectStorage` on GCP which did not allow to use `removeFileIfExists` effectively breaking approximately half of `remove` functionality. Automatic detection for `DeleteObjects` S3 API, that is not supported by GCS. This will allow to use GCS without explicit `support_batch_delete=0` in configuration. [#37882](https://github.com/ClickHouse/ClickHouse/pull/37882) ([Vladimir Chebotarev](https://github.com/excitoon)).
* Fix refcnt for unused MergeTree parts in SELECT queries (may defer parts removal). [#37913](https://github.com/ClickHouse/ClickHouse/pull/37913) ([Azat Khuzhin](https://github.com/azat)).
* Expose basic Keeper related monitoring data (via ProfileEvents and CurrentMetrics). [#38072](https://github.com/ClickHouse/ClickHouse/pull/38072) ([lingpeng0314](https://github.com/lingpeng0314)).
* Added kerberosInit function and corresponding KerberosInit class as a replacement for kinit executable. Replaced all calls of kinit in Kafka and HDFS code by call of kerberosInit function. Added new integration test. Closes [#27651](https://github.com/ClickHouse/ClickHouse/issues/27651). [#38105](https://github.com/ClickHouse/ClickHouse/pull/38105) ([Roman Vasin](https://github.com/rvasin)).
* * Add setting `multiple_joins_try_to_keep_original_names` to not rewrite identifier name on multiple JOINs rewrite, close [#34697](https://github.com/ClickHouse/ClickHouse/issues/34697). [#38149](https://github.com/ClickHouse/ClickHouse/pull/38149) ([Vladimir C](https://github.com/vdimir)).
* improved trace-visualizer UX. [#38169](https://github.com/ClickHouse/ClickHouse/pull/38169) ([Sergei Trifonov](https://github.com/serxa)).
* Add ability to pass headers to url table function / storage via sql. Closes [#37897](https://github.com/ClickHouse/ClickHouse/issues/37897). [#38176](https://github.com/ClickHouse/ClickHouse/pull/38176) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Enable trace collection for AArch64. [#38181](https://github.com/ClickHouse/ClickHouse/pull/38181) ([Maksim Kita](https://github.com/kitaisreal)).
* Do not skip symlinks in `user_defined` directory during SQL user defined functions loading. Closes [#38042](https://github.com/ClickHouse/ClickHouse/issues/38042). [#38184](https://github.com/ClickHouse/ClickHouse/pull/38184) ([Maksim Kita](https://github.com/kitaisreal)).
* Improve the stability for hive storage integration test. Move the data prepare step into test.py. [#38260](https://github.com/ClickHouse/ClickHouse/pull/38260) ([lgbo](https://github.com/lgbo-ustc)).
* Added background cleanup of subdirectories in `store/`. In some cases clickhouse-server might left garbage subdirectories in `store/` (for example, on unsuccessful table creation) and those dirs were never been removed. Fixes [#33710](https://github.com/ClickHouse/ClickHouse/issues/33710). [#38265](https://github.com/ClickHouse/ClickHouse/pull/38265) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Add `DESCRIBE CACHE` query to show cache settings from config. Add `SHOW CACHES` query to show available filesystem caches list. [#38279](https://github.com/ClickHouse/ClickHouse/pull/38279) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Add access check for system drop fs cache. Support ON CLUSTER. [#38319](https://github.com/ClickHouse/ClickHouse/pull/38319) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Support `auto_close` option for postgres engine connection. Closes [#31486](https://github.com/ClickHouse/ClickHouse/issues/31486). [#38363](https://github.com/ClickHouse/ClickHouse/pull/38363) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix PostgreSQL database engine incompatibility on upgrade from 21.3 to 22.3. Closes [#36659](https://github.com/ClickHouse/ClickHouse/issues/36659). [#38369](https://github.com/ClickHouse/ClickHouse/pull/38369) ([Kseniia Sumarokova](https://github.com/kssenii)).
* `filesystemAvailable` and similar functions now work in `clickhouse-local`. This closes [#38423](https://github.com/ClickHouse/ClickHouse/issues/38423). [#38424](https://github.com/ClickHouse/ClickHouse/pull/38424) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Hardware benchmark now has support for automatic results uploading. [#38427](https://github.com/ClickHouse/ClickHouse/pull/38427) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* The table `system.asynchronous_metric_log` is further optimized for storage space. This closes [#38134](https://github.com/ClickHouse/ClickHouse/issues/38134). See the [YouTube video](https://www.youtube.com/watch?v=0fSp9SF8N8A). [#38428](https://github.com/ClickHouse/ClickHouse/pull/38428) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Functions multiMatchAny(), multiMatchAnyIndex(), multiMatchAllIndices() and their fuzzy variants now accept non-const pattern array argument. [#38485](https://github.com/ClickHouse/ClickHouse/pull/38485) ([Robert Schulze](https://github.com/rschu1ze)).
* Added L2 Squared distance and norm for both arrays and tuples. [#38545](https://github.com/ClickHouse/ClickHouse/pull/38545) ([Julian Gilyadov](https://github.com/israelg99)).
* Add revision() function. [#38555](https://github.com/ClickHouse/ClickHouse/pull/38555) ([Azat Khuzhin](https://github.com/azat)).
* Add `group_by_use_nulls` setting to make aggregation key columns nullable in the case of ROLLUP, CUBE and GROUPING SETS. Closes [#37359](https://github.com/ClickHouse/ClickHouse/issues/37359). [#38642](https://github.com/ClickHouse/ClickHouse/pull/38642) ([Dmitry Novik](https://github.com/novikd)).
* Fix GCS via proxy tunnel usage. [#38726](https://github.com/ClickHouse/ClickHouse/pull/38726) ([Azat Khuzhin](https://github.com/azat)).
* Support `\i file` in clickhouse client / local (similar to psql \i). [#38813](https://github.com/ClickHouse/ClickHouse/pull/38813) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Allow null modifier in columns declaration for table functions. [#38816](https://github.com/ClickHouse/ClickHouse/pull/38816) ([Kruglov Pavel](https://github.com/Avogar)).
* - Deactivate `mutations_finalizing_task` before shutdown to avoid `TABLE_IS_READ_ONLY` errors. [#38851](https://github.com/ClickHouse/ClickHouse/pull/38851) ([Raúl Marín](https://github.com/Algunenano)).
* Fix waiting of shared lock after exclusive lock failure. [#38864](https://github.com/ClickHouse/ClickHouse/pull/38864) ([Azat Khuzhin](https://github.com/azat)).
* Add the ability to specify compression level during data export. [#38907](https://github.com/ClickHouse/ClickHouse/pull/38907) ([Nikolay Degterinsky](https://github.com/evillique)).
* New option `rewrite` in `EXPLAIN AST`. If enabled, it shows AST after it's rewritten, otherwise AST of original query. Disabled by default. [#38910](https://github.com/ClickHouse/ClickHouse/pull/38910) ([Igor Nikonov](https://github.com/devcrafter)).
* - Stop reporting Zookeeper "Node exists" exceptions in system.errors when they are expected. [#38961](https://github.com/ClickHouse/ClickHouse/pull/38961) ([Raúl Marín](https://github.com/Algunenano)).
* Allow to specify globs `* or {expr1, expr2, expr3}` inside a key for `clickhouse-extract-from-config` tool. [#38966](https://github.com/ClickHouse/ClickHouse/pull/38966) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
* Add option enabling that SELECT from the system database requires grant. Details:. [#38970](https://github.com/ClickHouse/ClickHouse/pull/38970) ([Vitaly Baranov](https://github.com/vitlibar)).
* - clearOldLogs: Don't report KEEPER_EXCEPTION on concurrent deletes. [#39016](https://github.com/ClickHouse/ClickHouse/pull/39016) ([Raúl Marín](https://github.com/Algunenano)).
* clickhouse-keeper improvement: persist metainformation about keeper servers to disk. [#39069](https://github.com/ClickHouse/ClickHouse/pull/39069) ([Antonio Andelic](https://github.com/antonio2368)).
* Continue without exception when running out of disk space when using filesystem cache. [#39106](https://github.com/ClickHouse/ClickHouse/pull/39106) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Handling SIGTERM signals from k8s. [#39130](https://github.com/ClickHouse/ClickHouse/pull/39130) ([Timur Solodovnikov](https://github.com/tsolodov)).
* SQL function multiStringAllPositions() now accepts non-const needle arguments. [#39167](https://github.com/ClickHouse/ClickHouse/pull/39167) ([Robert Schulze](https://github.com/rschu1ze)).
* Add merge_algorithm (Undecided, Horizontal, Vertical) to system.part_log. [#39181](https://github.com/ClickHouse/ClickHouse/pull/39181) ([Azat Khuzhin](https://github.com/azat)).
* Improve isNullable/isConstant/isNull/isNotNull performance for LowCardinality argument. [#39192](https://github.com/ClickHouse/ClickHouse/pull/39192) ([Kruglov Pavel](https://github.com/Avogar)).
* - Don't report system.errors when the disk is not rotational. [#39216](https://github.com/ClickHouse/ClickHouse/pull/39216) ([Raúl Marín](https://github.com/Algunenano)).
* Metric `result_bytes` for `INSERT` queries in `system.query_log` shows number of bytes inserted. Previously value was incorrect and stored the same value as `result_rows`. [#39225](https://github.com/ClickHouse/ClickHouse/pull/39225) ([Ilya Yatsishin](https://github.com/qoega)).
* The CPU usage metric in clickhouse-client will be displayed in a better way. Fixes [#38756](https://github.com/ClickHouse/ClickHouse/issues/38756). [#39280](https://github.com/ClickHouse/ClickHouse/pull/39280) ([Sergei Trifonov](https://github.com/serxa)).
* Rethrow exception on filesystem cache initialisation on server startup, better error message. [#39386](https://github.com/ClickHouse/ClickHouse/pull/39386) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Support milliseconds, microseconds and nanoseconds in `parseTimeDelta` function. [#39447](https://github.com/ClickHouse/ClickHouse/pull/39447) ([Kruglov Pavel](https://github.com/Avogar)).
#### Bug Fix
* Fix crash when executing GRANT ALL ON *.* with ON CLUSTER. It was broken in https://github.com/ClickHouse/ClickHouse/pull/35767. This closes [#38618](https://github.com/ClickHouse/ClickHouse/issues/38618). [#38674](https://github.com/ClickHouse/ClickHouse/pull/38674) ([Vitaly Baranov](https://github.com/vitlibar)).
* * Fixed crash caused by IHiveFile be shared among threads. [#38887](https://github.com/ClickHouse/ClickHouse/pull/38887) ([lgbo](https://github.com/lgbo-ustc)).
#### Build/Testing/Packaging Improvement
* - Apply Clang Thread Safety Analysis (TSA) annotations to ClickHouse. [#38068](https://github.com/ClickHouse/ClickHouse/pull/38068) ([Robert Schulze](https://github.com/rschu1ze)).
* - System table "system.licenses" is now correctly populated on Mac (Darwin). [#38294](https://github.com/ClickHouse/ClickHouse/pull/38294) ([Robert Schulze](https://github.com/rschu1ze)).
* Handle full queue exception in clickhouse-test. If it happened we need to collect debug info to understand what queries didn't finish. [#38490](https://github.com/ClickHouse/ClickHouse/pull/38490) ([Dmitry Novik](https://github.com/novikd)).
* - Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Add `clickhouse-diagnostics` binary to the packages. [#38647](https://github.com/ClickHouse/ClickHouse/pull/38647) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Always print stacktraces if test queue is full. Follow up [#38490](https://github.com/ClickHouse/ClickHouse/issues/38490) cc @tavplubix. [#38662](https://github.com/ClickHouse/ClickHouse/pull/38662) ([Dmitry Novik](https://github.com/novikd)).
* Align branches within a 32B boundary to make benchmark more stable. [#38988](https://github.com/ClickHouse/ClickHouse/pull/38988) ([Guo Wangyang](https://github.com/guowangy)).
* Fix LSan by fixing getauxval(). [#39299](https://github.com/ClickHouse/ClickHouse/pull/39299) ([Azat Khuzhin](https://github.com/azat)).
* Adapt universal installation script for FreeBSD. [#39302](https://github.com/ClickHouse/ClickHouse/pull/39302) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
* Fix projection exception when aggregation keys are wrapped inside other functions. This fixes [#37151](https://github.com/ClickHouse/ClickHouse/issues/37151). [#37155](https://github.com/ClickHouse/ClickHouse/pull/37155) ([Amos Bird](https://github.com/amosbird)).
* Fix possible logical error `... with argument with type Nothing and default implementation for Nothing is expected to return result with type Nothing, got ...` in some functions. Closes: [#37610](https://github.com/ClickHouse/ClickHouse/issues/37610) Closes: [#37741](https://github.com/ClickHouse/ClickHouse/issues/37741). [#37759](https://github.com/ClickHouse/ClickHouse/pull/37759) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix incorrect columns order in subqueries of UNION (in case of duplicated columns in subselects may produce incorrect result). [#37887](https://github.com/ClickHouse/ClickHouse/pull/37887) ([Azat Khuzhin](https://github.com/azat)).
* Fix incorrect work of MODIFY ALTER Column with column names that contain dots. Closes [#37907](https://github.com/ClickHouse/ClickHouse/issues/37907). [#37971](https://github.com/ClickHouse/ClickHouse/pull/37971) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix reading of sparse columns from `MergeTree` tables that store their data in S3. [#37978](https://github.com/ClickHouse/ClickHouse/pull/37978) ([Anton Popov](https://github.com/CurtizJ)).
* Fix rounding for `Decimal128/Decimal256` with more than 19-digits long scale. [#38027](https://github.com/ClickHouse/ClickHouse/pull/38027) ([Igor Nikonov](https://github.com/devcrafter)).
* Fix possible crash in `Distributed` async insert in case of removing a replica from config. [#38029](https://github.com/ClickHouse/ClickHouse/pull/38029) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix "Missing columns" for GLOBAL JOIN with CTE w/o alias. [#38056](https://github.com/ClickHouse/ClickHouse/pull/38056) ([Azat Khuzhin](https://github.com/azat)).
* Rewrite tuple functions as literals in backwards-compatibility mode. [#38096](https://github.com/ClickHouse/ClickHouse/pull/38096) ([Anton Kozlov](https://github.com/tonickkozlov)).
* - Fix redundant memory reservation for output block during `ORDER BY`. [#38127](https://github.com/ClickHouse/ClickHouse/pull/38127) ([iyupeng](https://github.com/iyupeng)).
* Fix possible logical error `Bad cast from type DB::IColumn* to DB::ColumnNullable*` in array mapped functions. Closes [#38006](https://github.com/ClickHouse/ClickHouse/issues/38006). [#38132](https://github.com/ClickHouse/ClickHouse/pull/38132) ([Kruglov Pavel](https://github.com/Avogar)).
* * Fix temporary name clash in partial merge join, close [#37928](https://github.com/ClickHouse/ClickHouse/issues/37928). [#38135](https://github.com/ClickHouse/ClickHouse/pull/38135) ([Vladimir C](https://github.com/vdimir)).
* With table ```SQL CREATE TABLE nested_name_tuples ( `a` Tuple(x String, y Tuple(i Int32, j String)) ) ENGINE = Memory; ```. [#38136](https://github.com/ClickHouse/ClickHouse/pull/38136) ([lgbo](https://github.com/lgbo-ustc)).
* Fix bug with nested short-circuit functions that led to execution of arguments even if condition is false. Closes [#38040](https://github.com/ClickHouse/ClickHouse/issues/38040). [#38173](https://github.com/ClickHouse/ClickHouse/pull/38173) ([Kruglov Pavel](https://github.com/Avogar)).
* (Window View is a experimental feature) Fix LOGICAL_ERROR for WINDOW VIEW with incorrect structure. [#38205](https://github.com/ClickHouse/ClickHouse/pull/38205) ([Azat Khuzhin](https://github.com/azat)).
* Update librdkafka submodule to fix crash when an OAUTHBEARER refresh callback is set. [#38225](https://github.com/ClickHouse/ClickHouse/pull/38225) ([Rafael Acevedo](https://github.com/racevedoo)).
* Do not allow recursive usage of OvercommitTracker during logging. Fixes [#37794](https://github.com/ClickHouse/ClickHouse/issues/37794) cc @tavplubix @davenger. [#38246](https://github.com/ClickHouse/ClickHouse/pull/38246) ([Dmitry Novik](https://github.com/novikd)).
* Fix INSERT into Distributed hung due to ProfileEvents. [#38307](https://github.com/ClickHouse/ClickHouse/pull/38307) ([Azat Khuzhin](https://github.com/azat)).
* Fix retries in PostgreSQL engine. [#38310](https://github.com/ClickHouse/ClickHouse/pull/38310) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix optimization in PartialSortingTransform (SIGSEGV and possible incorrect result). [#38324](https://github.com/ClickHouse/ClickHouse/pull/38324) ([Azat Khuzhin](https://github.com/azat)).
* Fix RabbitMQ with formats based on PeekableReadBuffer. Closes [#38061](https://github.com/ClickHouse/ClickHouse/issues/38061). [#38356](https://github.com/ClickHouse/ClickHouse/pull/38356) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix possible `Invalid number of rows in Chunk` in materialised pg. Closes [#37323](https://github.com/ClickHouse/ClickHouse/issues/37323). [#38360](https://github.com/ClickHouse/ClickHouse/pull/38360) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix RabbitMQ configuration with connection string setting. Closes [#36531](https://github.com/ClickHouse/ClickHouse/issues/36531). [#38365](https://github.com/ClickHouse/ClickHouse/pull/38365) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix PostgreSQL engine not using PostgreSQL schema when retrieving array dimension size. Closes [#36755](https://github.com/ClickHouse/ClickHouse/issues/36755). Closes [#36772](https://github.com/ClickHouse/ClickHouse/issues/36772). [#38366](https://github.com/ClickHouse/ClickHouse/pull/38366) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix incorrect result of distributed queries with `DISTINCT` and `LIMIT`. Fixes [#38282](https://github.com/ClickHouse/ClickHouse/issues/38282). [#38371](https://github.com/ClickHouse/ClickHouse/pull/38371) ([Anton Popov](https://github.com/CurtizJ)).
* fix: expose new CH keeper port in Dockerfile clickhouse/clickhouse-keeper fix: use correct KEEPER_CONFIG filename in clickhouse/clickhouse-keeper docker image. [#38462](https://github.com/ClickHouse/ClickHouse/pull/38462) ([Evgeny Kruglov](https://github.com/nordluf)).
* Fix parts removal (will be left forever if they had not been removed on server shutdown) after incorrect server shutdown. [#38486](https://github.com/ClickHouse/ClickHouse/pull/38486) ([Azat Khuzhin](https://github.com/azat)).
* Fixes [#38498](https://github.com/ClickHouse/ClickHouse/issues/38498) Current Implementation is similar to what shell does mentiond by @rschu1ze [here](https://github.com/ClickHouse/ClickHouse/pull/38502#issuecomment-1169057723). [#38502](https://github.com/ClickHouse/ClickHouse/pull/38502) ([Heena Bansal](https://github.com/HeenaBansal2009)).
* Fix table creation to avoid replication issues with pre-22.4 replicas. [#38541](https://github.com/ClickHouse/ClickHouse/pull/38541) ([Raúl Marín](https://github.com/Algunenano)).
* Fix crash for `mapUpdate`, `mapFilter` functions when using with constant map argument. Closes [#38547](https://github.com/ClickHouse/ClickHouse/issues/38547). [#38553](https://github.com/ClickHouse/ClickHouse/pull/38553) ([hexiaoting](https://github.com/hexiaoting)).
* Fix wrong results of countSubstrings() & position() on patterns with 0-bytes. [#38589](https://github.com/ClickHouse/ClickHouse/pull/38589) ([Robert Schulze](https://github.com/rschu1ze)).
* Now it's possible to start a clickhouse-server and attach/detach tables even for tables with the incorrect values of IPv4/IPv6 representation. Proper fix for issue [#35156](https://github.com/ClickHouse/ClickHouse/issues/35156). [#38590](https://github.com/ClickHouse/ClickHouse/pull/38590) ([alesapin](https://github.com/alesapin)).
* Adapt some more nodes to avoid issues with pre-22.4 replicas. [#38627](https://github.com/ClickHouse/ClickHouse/pull/38627) ([Raúl Marín](https://github.com/Algunenano)).
* Fix toHour() monotonicity which can lead to incorrect query result (incorrect index analysis). This fixes [#38333](https://github.com/ClickHouse/ClickHouse/issues/38333). [#38675](https://github.com/ClickHouse/ClickHouse/pull/38675) ([Amos Bird](https://github.com/amosbird)).
* `rankCorr` function will work correctly if some arguments are NaNs. This closes [#38396](https://github.com/ClickHouse/ClickHouse/issues/38396). [#38722](https://github.com/ClickHouse/ClickHouse/pull/38722) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Fix `parallel_view_processing=1` with `optimize_trivial_insert_select=1`. Fix `max_insert_threads` while pushing to views. [#38731](https://github.com/ClickHouse/ClickHouse/pull/38731) ([Azat Khuzhin](https://github.com/azat)).
* Fix use-after-free for Map combinator that leads to incorrect result. [#38748](https://github.com/ClickHouse/ClickHouse/pull/38748) ([Azat Khuzhin](https://github.com/azat)).
* Fix throwing exception for seekable read from s3 (exception was not thrown). [#38773](https://github.com/ClickHouse/ClickHouse/pull/38773) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix checking whether s3 storage support parallel writes. It resulted in s3 parallel writes not working. [#38792](https://github.com/ClickHouse/ClickHouse/pull/38792) ([chen](https://github.com/xiedeyantu)).
* Fix s3 seekable reads with parallel read buffer. (Affected memory usage during query). Closes [#38258](https://github.com/ClickHouse/ClickHouse/issues/38258). [#38802](https://github.com/ClickHouse/ClickHouse/pull/38802) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Update `simdjson`. This fixes [#38621](https://github.com/ClickHouse/ClickHouse/issues/38621). [#38838](https://github.com/ClickHouse/ClickHouse/pull/38838) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* MergeTree fix possible logical error for Vertical merges. [#38859](https://github.com/ClickHouse/ClickHouse/pull/38859) ([Maksim Kita](https://github.com/kitaisreal)).
* - Fix settings profile with seconds unit. [#38896](https://github.com/ClickHouse/ClickHouse/pull/38896) ([Raúl Marín](https://github.com/Algunenano)).
* Fix incorrect partition pruning when there is a nullable partition. This fixes [#38941](https://github.com/ClickHouse/ClickHouse/issues/38941). [#38946](https://github.com/ClickHouse/ClickHouse/pull/38946) ([Amos Bird](https://github.com/amosbird)).
* Fix fsync_part_directory for fetches. [#38993](https://github.com/ClickHouse/ClickHouse/pull/38993) ([Azat Khuzhin](https://github.com/azat)).
* Functions multiMatch[Fuzzy](AllIndices/Any/AnyIndex)() no throw a logical error if the needle argument is empty. [#39012](https://github.com/ClickHouse/ClickHouse/pull/39012) ([Robert Schulze](https://github.com/rschu1ze)).
* Any allocations inside OvercommitTracker may lead to deadlock. Logging was not very informative so it's easier just to remove logging. Fixes [#37794](https://github.com/ClickHouse/ClickHouse/issues/37794). [#39030](https://github.com/ClickHouse/ClickHouse/pull/39030) ([Dmitry Novik](https://github.com/novikd)).
* Fix toHour() monotonicity which can lead to incorrect query result (incorrect index analysis). This fixes [#38333](https://github.com/ClickHouse/ClickHouse/issues/38333). [#39037](https://github.com/ClickHouse/ClickHouse/pull/39037) ([Amos Bird](https://github.com/amosbird)).
* Fix bug in filesystem cache that could happen in some corner case which coincided with cache capacity hitting the limit. Closes [#39066](https://github.com/ClickHouse/ClickHouse/issues/39066). [#39070](https://github.com/ClickHouse/ClickHouse/pull/39070) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix ActionsDAG construction for arguments of window expressions. Fixes [#38538](https://github.com/ClickHouse/ClickHouse/issues/38538) Allow using of higher-order functions in window expressions. [#39112](https://github.com/ClickHouse/ClickHouse/pull/39112) ([Dmitry Novik](https://github.com/novikd)).
* Keep `LowCardinality` type in `tuple()` function. Previously `LowCardinality` type was dropped and elements of created tuple had underlying type of `LowCardinality`. [#39113](https://github.com/ClickHouse/ClickHouse/pull/39113) ([Anton Popov](https://github.com/CurtizJ)).
* Fix error `Block structure mismatch` which could happen for INSERT into table with attached MATERIALIZED VIEW and enabled setting `extremes = 1`. Closes [#29759](https://github.com/ClickHouse/ClickHouse/issues/29759) and [#38729](https://github.com/ClickHouse/ClickHouse/issues/38729). [#39125](https://github.com/ClickHouse/ClickHouse/pull/39125) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix unexpected query result when both `optimize_trivial_count_query` and `empty_result_for_aggregation_by_empty_set` are set to true. This fixes [#39140](https://github.com/ClickHouse/ClickHouse/issues/39140). [#39155](https://github.com/ClickHouse/ClickHouse/pull/39155) ([Amos Bird](https://github.com/amosbird)).
* Fixed error `Not found column Type in block` in selects with `PREWHERE` and read-in-order optimizations. [#39157](https://github.com/ClickHouse/ClickHouse/pull/39157) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Fix extremely rare race condition in during hardnlinks for remote fs. The only way to reproduce it is concurrent run of backups. [#39190](https://github.com/ClickHouse/ClickHouse/pull/39190) ([alesapin](https://github.com/alesapin)).
* Fix fetch of in-memory part with `allow_remote_fs_zero_copy_replication`. [#39214](https://github.com/ClickHouse/ClickHouse/pull/39214) ([Azat Khuzhin](https://github.com/azat)).
* Fix NOEXCEPT_SCOPE (before it calls std::terminate and looses the exception). [#39229](https://github.com/ClickHouse/ClickHouse/pull/39229) ([Azat Khuzhin](https://github.com/azat)).
* Declare RabbitMQ queue without default arguments `x-max-length` and `x-overflow`. [#39259](https://github.com/ClickHouse/ClickHouse/pull/39259) ([rnbondarenko](https://github.com/rnbondarenko)).
* Fix segmentation fault in MaterializedPostgreSQL database engine, which could happen if some exception occurred at replication initialisation. Closes [#36939](https://github.com/ClickHouse/ClickHouse/issues/36939). [#39272](https://github.com/ClickHouse/ClickHouse/pull/39272) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix incorrect fetch postgresql tables query fro PostgreSQL database engine. Closes [#33502](https://github.com/ClickHouse/ClickHouse/issues/33502). [#39283](https://github.com/ClickHouse/ClickHouse/pull/39283) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix possible UB in MergeTreeBackgroundExecutor (leads to SIGSEGV on race with DROP/DETACH). [#39342](https://github.com/ClickHouse/ClickHouse/pull/39342) ([Azat Khuzhin](https://github.com/azat)).
* Avoid possible abort() in CapnProto on exception descruction. Closes [#30706](https://github.com/ClickHouse/ClickHouse/issues/30706). [#39365](https://github.com/ClickHouse/ClickHouse/pull/39365) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix behaviour of dictHas for direct dictionaries when multiple lookups to the same key are made in a single action. [#39385](https://github.com/ClickHouse/ClickHouse/pull/39385) ([James Morrison](https://github.com/jawm)).
* Fix crash which may happen while reading from dictionary with `DateTime64` attribute. Fixes [#38930](https://github.com/ClickHouse/ClickHouse/issues/38930). [#39391](https://github.com/ClickHouse/ClickHouse/pull/39391) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix WriteBuffer finalize in destructor when cacnel query that could lead to stuck query or even terminate. Closes [#38199](https://github.com/ClickHouse/ClickHouse/issues/38199). [#39396](https://github.com/ClickHouse/ClickHouse/pull/39396) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix UB (stack-use-after-scope) in extactAll(). [#39397](https://github.com/ClickHouse/ClickHouse/pull/39397) ([Azat Khuzhin](https://github.com/azat)).
* Fix incorrect query result when trivial count optimization is in effect with array join. This fixes [#39431](https://github.com/ClickHouse/ClickHouse/issues/39431). [#39444](https://github.com/ClickHouse/ClickHouse/pull/39444) ([Amos Bird](https://github.com/amosbird)).
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
* Disable send_logs_level for INSERT into Distributed to avoid possible hung. [#35075](https://github.com/ClickHouse/ClickHouse/pull/35075) ([Azat Khuzhin](https://github.com/azat)).
#### NO CL ENTRY
* NO CL ENTRY: 'Revert "Add a setting to use more memory for zstd decompression"'. [#38194](https://github.com/ClickHouse/ClickHouse/pull/38194) ([alesapin](https://github.com/alesapin)).
* NO CL ENTRY: 'Revert "Revert "Add a setting to use more memory for zstd decompression""'. [#38196](https://github.com/ClickHouse/ClickHouse/pull/38196) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* NO CL ENTRY: 'Revert "ClickHouse's boringssl module updated to the official version of the FIPS compliant."'. [#38201](https://github.com/ClickHouse/ClickHouse/pull/38201) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* NO CL ENTRY: 'Revert "Fix optimization in PartialSortingTransform (SIGSEGV and possible incorrect result)"'. [#38361](https://github.com/ClickHouse/ClickHouse/pull/38361) ([Alexander Tokmakov](https://github.com/tavplubix)).
* NO CL ENTRY: 'Revert "Add support for io_uring read method"'. [#38377](https://github.com/ClickHouse/ClickHouse/pull/38377) ([Alexander Tokmakov](https://github.com/tavplubix)).
* NO CL ENTRY: 'Revert "Revert "Fix optimization in PartialSortingTransform (SIGSEGV and possible incorrect result)""'. [#38449](https://github.com/ClickHouse/ClickHouse/pull/38449) ([Maksim Kita](https://github.com/kitaisreal)).
* NO CL ENTRY: 'Don't spoil return code of integration tests runner with redundant tee'. [#38548](https://github.com/ClickHouse/ClickHouse/pull/38548) ([Vladimir Chebotarev](https://github.com/excitoon)).
* NO CL ENTRY: 'Revert "Non Negative Derivative window function"'. [#38551](https://github.com/ClickHouse/ClickHouse/pull/38551) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* NO CL ENTRY: 'Revert "Upload to S3 compressed self-extracting clickhouse"'. [#38788](https://github.com/ClickHouse/ClickHouse/pull/38788) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* NO CL ENTRY: 'Revert "Smallish updates of dev guide"'. [#38848](https://github.com/ClickHouse/ClickHouse/pull/38848) ([Alexander Tokmakov](https://github.com/tavplubix)).
* NO CL ENTRY: 'Revert "Fix toHour() monotonicity which can lead to incorrect query result (incorrect index analysis)"'. [#39001](https://github.com/ClickHouse/ClickHouse/pull/39001) ([Alexander Tokmakov](https://github.com/tavplubix)).
* NO CL ENTRY: 'Revert "Fix WriteBuffer finalize in destructor when cacnel query"'. [#39433](https://github.com/ClickHouse/ClickHouse/pull/39433) ([Kruglov Pavel](https://github.com/Avogar)).
* NO CL ENTRY: 'Revert "[RFC] Fix LSan by fixing getauxval()"'. [#39434](https://github.com/ClickHouse/ClickHouse/pull/39434) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* NO CL ENTRY: 'Revert "Remove broken optimisation in Direct dictionary dictHas implementation"'. [#39461](https://github.com/ClickHouse/ClickHouse/pull/39461) ([Alexander Tokmakov](https://github.com/tavplubix)).
* NO CL ENTRY: 'Revert "Fix trivial count optimization with array join"'. [#39466](https://github.com/ClickHouse/ClickHouse/pull/39466) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Separate data storage abstraction for MergeTree [#36555](https://github.com/ClickHouse/ClickHouse/pull/36555) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Randomize settings related to in-order read/aggregation [#36914](https://github.com/ClickHouse/ClickHouse/pull/36914) ([Azat Khuzhin](https://github.com/azat)).
* Merge tree reader support for multiple read/filter steps: row level filter, prewhere, ... [#37165](https://github.com/ClickHouse/ClickHouse/pull/37165) ([Alexander Gololobov](https://github.com/davenger)).
* Backup Improvements 6 [#37358](https://github.com/ClickHouse/ClickHouse/pull/37358) ([Vitaly Baranov](https://github.com/vitlibar)).
* Move `updateInputStream` to `ITransformingStep` [#37393](https://github.com/ClickHouse/ClickHouse/pull/37393) ([Nikita Taranov](https://github.com/nickitat)).
* Proper wait of the clickhouse-server in tests [#37560](https://github.com/ClickHouse/ClickHouse/pull/37560) ([Azat Khuzhin](https://github.com/azat)).
* Upgrade curl to 7.83.1 [#37795](https://github.com/ClickHouse/ClickHouse/pull/37795) ([Suzy Wang](https://github.com/SuzyWangIBMer)).
* Try fix flaky tests with transactions [#37822](https://github.com/ClickHouse/ClickHouse/pull/37822) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Send perf tests results to ci database [#37841](https://github.com/ClickHouse/ClickHouse/pull/37841) ([Vladimir C](https://github.com/vdimir)).
* Remove duplicate peak mem log [#37860](https://github.com/ClickHouse/ClickHouse/pull/37860) ([Amos Bird](https://github.com/amosbird)).
* tests: fix log_comment (extra quotes) [#37932](https://github.com/ClickHouse/ClickHouse/pull/37932) ([Azat Khuzhin](https://github.com/azat)).
* Throw exception when xml user profile does not exist [#38024](https://github.com/ClickHouse/ClickHouse/pull/38024) ([nvartolomei](https://github.com/nvartolomei)).
* Add `SYNC` command to internal ZooKeeper client [#38047](https://github.com/ClickHouse/ClickHouse/pull/38047) ([Antonio Andelic](https://github.com/antonio2368)).
* Better support of GCP storage [#38069](https://github.com/ClickHouse/ClickHouse/pull/38069) ([Anton Popov](https://github.com/CurtizJ)).
* Build artifacts upload [#38086](https://github.com/ClickHouse/ClickHouse/pull/38086) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Extract some diff from pr [#36171](https://github.com/ClickHouse/ClickHouse/issues/36171) [#38088](https://github.com/ClickHouse/ClickHouse/pull/38088) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Aggregate functions added restrict into batch methods [#38090](https://github.com/ClickHouse/ClickHouse/pull/38090) ([Maksim Kita](https://github.com/kitaisreal)).
* Add perf checkers to all Jepsen tests [#38091](https://github.com/ClickHouse/ClickHouse/pull/38091) ([Antonio Andelic](https://github.com/antonio2368)).
* Some fixes for tests with tsan [#38106](https://github.com/ClickHouse/ClickHouse/pull/38106) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Bring back [#36396](https://github.com/ClickHouse/ClickHouse/issues/36396) [#38110](https://github.com/ClickHouse/ClickHouse/pull/38110) ([Nikita Taranov](https://github.com/nickitat)).
* More suppressions for backward compatibility check [#38131](https://github.com/ClickHouse/ClickHouse/pull/38131) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Cherry pick [#38137](https://github.com/ClickHouse/ClickHouse/pull/38137) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Disable parameters for non direct executable user defined functions [#38142](https://github.com/ClickHouse/ClickHouse/pull/38142) ([Maksim Kita](https://github.com/kitaisreal)).
* SortDescription compile fix typo [#38144](https://github.com/ClickHouse/ClickHouse/pull/38144) ([Maksim Kita](https://github.com/kitaisreal)).
* Update version after release [#38147](https://github.com/ClickHouse/ClickHouse/pull/38147) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* New changelog and versions updated [#38148](https://github.com/ClickHouse/ClickHouse/pull/38148) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Some fixes for clickhouse-disks [#38150](https://github.com/ClickHouse/ClickHouse/pull/38150) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Remove processor description from span attributes - it is not working [#38157](https://github.com/ClickHouse/ClickHouse/pull/38157) ([Ilya Yatsishin](https://github.com/qoega)).
* Bump minimum / maximum LLVM to 12 / 14 [#38170](https://github.com/ClickHouse/ClickHouse/pull/38170) ([Robert Schulze](https://github.com/rschu1ze)).
* Disk transaction [#38182](https://github.com/ClickHouse/ClickHouse/pull/38182) ([alesapin](https://github.com/alesapin)).
* Check row size to avoid out of bounds access in PostgreSQLSource [#38190](https://github.com/ClickHouse/ClickHouse/pull/38190) ([Alexander Gololobov](https://github.com/davenger)).
* tests: add no-backward-compatibility-check for 02067_lost_part_s3 [#38195](https://github.com/ClickHouse/ClickHouse/pull/38195) ([Azat Khuzhin](https://github.com/azat)).
* tests/stress: fix TSan detection (enables thread fuzzer for non-TSan builds) [#38207](https://github.com/ClickHouse/ClickHouse/pull/38207) ([Azat Khuzhin](https://github.com/azat)).
* tests: disable 01646_system_restart_replicas_smoke under stress tests [#38212](https://github.com/ClickHouse/ClickHouse/pull/38212) ([Azat Khuzhin](https://github.com/azat)).
* tests/stress: fix TSan detection [#38213](https://github.com/ClickHouse/ClickHouse/pull/38213) ([Azat Khuzhin](https://github.com/azat)).
* buffer's getFileSize small changes [#38227](https://github.com/ClickHouse/ClickHouse/pull/38227) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix test for system table count in diag tool [#38236](https://github.com/ClickHouse/ClickHouse/pull/38236) ([Dale McDiarmid](https://github.com/gingerwizard)).
* Update version_date.tsv after v22.3.7.28-lts [#38237](https://github.com/ClickHouse/ClickHouse/pull/38237) ([github-actions[bot]](https://github.com/apps/github-actions)).
* Changelog attrs [#38238](https://github.com/ClickHouse/ClickHouse/pull/38238) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Fix reading from s3 in some corner cases [#38239](https://github.com/ClickHouse/ClickHouse/pull/38239) ([Anton Popov](https://github.com/CurtizJ)).
* use utility methods to access x509 struct fields. [#38251](https://github.com/ClickHouse/ClickHouse/pull/38251) ([larryluogit](https://github.com/larryluogit)).
* Don't try to kill empty list of containers in `integration/runner` II [#38269](https://github.com/ClickHouse/ClickHouse/pull/38269) ([Vladimir Chebotarev](https://github.com/excitoon)).
* Improve runners AMI and init scripts [#38273](https://github.com/ClickHouse/ClickHouse/pull/38273) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Update thrift to 0.16.0 [#38280](https://github.com/ClickHouse/ClickHouse/pull/38280) ([Suzy Wang](https://github.com/SuzyWangIBMer)).
* Extract some diff from [#36171](https://github.com/ClickHouse/ClickHouse/issues/36171) [#38285](https://github.com/ClickHouse/ClickHouse/pull/38285) ([Kseniia Sumarokova](https://github.com/kssenii)).
* fix trace-viz zoom anomalies [#38287](https://github.com/ClickHouse/ClickHouse/pull/38287) ([Sergei Trifonov](https://github.com/serxa)).
* Integration tests volume [#38291](https://github.com/ClickHouse/ClickHouse/pull/38291) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* fix typo in view.md [#38292](https://github.com/ClickHouse/ClickHouse/pull/38292) ([Anton Petrov](https://github.com/gsenseless)).
* Backup improvements 7 [#38299](https://github.com/ClickHouse/ClickHouse/pull/38299) ([Vitaly Baranov](https://github.com/vitlibar)).
* Document why the submodule check does not halt the configuration [#38304](https://github.com/ClickHouse/ClickHouse/pull/38304) ([Robert Schulze](https://github.com/rschu1ze)).
* Fix misleading error message while s3 schema inference [#38306](https://github.com/ClickHouse/ClickHouse/pull/38306) ([Kruglov Pavel](https://github.com/Avogar)).
* Update README.md [#38313](https://github.com/ClickHouse/ClickHouse/pull/38313) ([Yuko Takagi](https://github.com/yukotakagi)).
* Ban projections for zero-copy replication in a right way [#38322](https://github.com/ClickHouse/ClickHouse/pull/38322) ([alesapin](https://github.com/alesapin)).
* Checkout full repositories for performance tests [#38327](https://github.com/ClickHouse/ClickHouse/pull/38327) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Fixed comments [#38331](https://github.com/ClickHouse/ClickHouse/pull/38331) ([Vladimir Chebotarev](https://github.com/excitoon)).
* Try to fix 02305_schema_inference_with_globs [#38337](https://github.com/ClickHouse/ClickHouse/pull/38337) ([Kruglov Pavel](https://github.com/Avogar)).
* Extend ZooKeeper list request with support for filtering persistent or ephemeral nodes only [#38338](https://github.com/ClickHouse/ClickHouse/pull/38338) ([Antonio Andelic](https://github.com/antonio2368)).
* Upload logs for getting all tests command [#38343](https://github.com/ClickHouse/ClickHouse/pull/38343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Followup test fix for ban projections [#38351](https://github.com/ClickHouse/ClickHouse/pull/38351) ([alesapin](https://github.com/alesapin)).
* Added --recursive to clickhouse-disks list [#38354](https://github.com/ClickHouse/ClickHouse/pull/38354) ([Alexander Gololobov](https://github.com/davenger)).
* Adding TLS V13 Test [#38355](https://github.com/ClickHouse/ClickHouse/pull/38355) ([larryluogit](https://github.com/larryluogit)).
* Better exception messages on wrong table engines/functions argument types [#38362](https://github.com/ClickHouse/ClickHouse/pull/38362) ([Kruglov Pavel](https://github.com/Avogar)).
* Better error message for failed odbc query [#38364](https://github.com/ClickHouse/ClickHouse/pull/38364) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Simplify parts commit methods [#38380](https://github.com/ClickHouse/ClickHouse/pull/38380) ([alesapin](https://github.com/alesapin)).
* Update docker-compose to try get rid of v1 errors [#38394](https://github.com/ClickHouse/ClickHouse/pull/38394) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Correct submodule after "base-x" commit [#38414](https://github.com/ClickHouse/ClickHouse/pull/38414) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Better hardware benchmark [#38419](https://github.com/ClickHouse/ClickHouse/pull/38419) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Refactoring to enable multi-match functions with non-const needles [#38434](https://github.com/ClickHouse/ClickHouse/pull/38434) ([Robert Schulze](https://github.com/rschu1ze)).
* more consistent work with paths in object storages [#38436](https://github.com/ClickHouse/ClickHouse/pull/38436) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Distinct sorted: calculate column positions once [#38438](https://github.com/ClickHouse/ClickHouse/pull/38438) ([Igor Nikonov](https://github.com/devcrafter)).
* Small improvement of the error message to hint at possible issue [#38458](https://github.com/ClickHouse/ClickHouse/pull/38458) ([Miel Donkers](https://github.com/mdonkers)).
* Fix comment [#38465](https://github.com/ClickHouse/ClickHouse/pull/38465) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Follow up for [#38436](https://github.com/ClickHouse/ClickHouse/issues/38436) [#38466](https://github.com/ClickHouse/ClickHouse/pull/38466) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Add 22.7 release webinar. [#38481](https://github.com/ClickHouse/ClickHouse/pull/38481) ([Yuko Takagi](https://github.com/yukotakagi)).
* Add some TSA annotations [#38487](https://github.com/ClickHouse/ClickHouse/pull/38487) ([Alexander Tokmakov](https://github.com/tavplubix)).
* tests: cleanup tmp data in 02335_column_ttl_expired_column_optimization [#38488](https://github.com/ClickHouse/ClickHouse/pull/38488) ([Azat Khuzhin](https://github.com/azat)).
* Cleanup: local clang-tidy warnings founded during review [#38489](https://github.com/ClickHouse/ClickHouse/pull/38489) ([Igor Nikonov](https://github.com/devcrafter)).
* Fix some clang-tidy warnings in headers [#38491](https://github.com/ClickHouse/ClickHouse/pull/38491) ([Robert Schulze](https://github.com/rschu1ze)).
* A tiny improvement in report logging [#38507](https://github.com/ClickHouse/ClickHouse/pull/38507) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* tests: fix 02305_schema_inference_with_globs flakiness [#38511](https://github.com/ClickHouse/ClickHouse/pull/38511) ([Azat Khuzhin](https://github.com/azat)).
* Try to fix flaky test [#38516](https://github.com/ClickHouse/ClickHouse/pull/38516) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix `_csv.Error: field larger than field limit` [#38518](https://github.com/ClickHouse/ClickHouse/pull/38518) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix application errors grep in stress test [#38520](https://github.com/ClickHouse/ClickHouse/pull/38520) ([Kruglov Pavel](https://github.com/Avogar)).
* Use of disk batch operations in MergeTree [#38531](https://github.com/ClickHouse/ClickHouse/pull/38531) ([alesapin](https://github.com/alesapin)).
* Backup Improvements 8 [#38537](https://github.com/ClickHouse/ClickHouse/pull/38537) ([Vitaly Baranov](https://github.com/vitlibar)).
* Update poco [#38540](https://github.com/ClickHouse/ClickHouse/pull/38540) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Don't use std::unique_lock unless we need to [#38542](https://github.com/ClickHouse/ClickHouse/pull/38542) ([Robert Schulze](https://github.com/rschu1ze)).
* Rename slightly weirdly named "BuilderBinTidy" to "BuilderBinClangTidy" [#38546](https://github.com/ClickHouse/ClickHouse/pull/38546) ([Robert Schulze](https://github.com/rschu1ze)).
* Don't rollback SessionID request in Keeper [#38556](https://github.com/ClickHouse/ClickHouse/pull/38556) ([Antonio Andelic](https://github.com/antonio2368)).
* Add logging in Epoll and TimerDescriptor in case of EINTR [#38559](https://github.com/ClickHouse/ClickHouse/pull/38559) ([Kruglov Pavel](https://github.com/Avogar)).
* SQL create drop index minor fixes [#38561](https://github.com/ClickHouse/ClickHouse/pull/38561) ([Maksim Kita](https://github.com/kitaisreal)).
* Update version_date.tsv and changelogs after v22.6.2.12-stable [#38563](https://github.com/ClickHouse/ClickHouse/pull/38563) ([github-actions[bot]](https://github.com/apps/github-actions)).
* Allow Ordinary database in Stress Tests [#38568](https://github.com/ClickHouse/ClickHouse/pull/38568) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Make postgres integration test great again [#38582](https://github.com/ClickHouse/ClickHouse/pull/38582) ([Ilya Yatsishin](https://github.com/qoega)).
* Add check for empty proccessors in AggregatingTransform::expandPipeline [#38584](https://github.com/ClickHouse/ClickHouse/pull/38584) ([filimonov](https://github.com/filimonov)).
* quick fix for 02112_with_fill_interval [#38587](https://github.com/ClickHouse/ClickHouse/pull/38587) ([Nikita Taranov](https://github.com/nickitat)).
* Remove zlib in mariadb-connector-c [#38599](https://github.com/ClickHouse/ClickHouse/pull/38599) ([Suzy Wang](https://github.com/SuzyWangIBMer)).
* Dictionaries added TSA annotations [#38601](https://github.com/ClickHouse/ClickHouse/pull/38601) ([Maksim Kita](https://github.com/kitaisreal)).
* CacheDictionary simplify update queue [#38602](https://github.com/ClickHouse/ClickHouse/pull/38602) ([Maksim Kita](https://github.com/kitaisreal)).
* Add separate option to omit symbols from heavy contrib [#38617](https://github.com/ClickHouse/ClickHouse/pull/38617) ([Azat Khuzhin](https://github.com/azat)).
* Fix exception messages in clickhouse su [#38619](https://github.com/ClickHouse/ClickHouse/pull/38619) ([filimonov](https://github.com/filimonov)).
* Added Greenplum benchmark [#38622](https://github.com/ClickHouse/ClickHouse/pull/38622) ([Dmitry Pavlov](https://github.com/kapustor)).
* Fix typo [#38623](https://github.com/ClickHouse/ClickHouse/pull/38623) ([tiegen](https://github.com/loyispa)).
* Better diagnostics in ReplicatedMergeTreeQueue [#38641](https://github.com/ClickHouse/ClickHouse/pull/38641) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Clean out randomized integration volumes each run [#38644](https://github.com/ClickHouse/ClickHouse/pull/38644) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Update README.md [#38651](https://github.com/ClickHouse/ClickHouse/pull/38651) ([Yuko Takagi](https://github.com/yukotakagi)).
* Better naming for stuff related to splitted debug symbols [#38654](https://github.com/ClickHouse/ClickHouse/pull/38654) ([Robert Schulze](https://github.com/rschu1ze)).
* Add test for keeper `mntr` command [#38656](https://github.com/ClickHouse/ClickHouse/pull/38656) ([alesapin](https://github.com/alesapin)).
* Update hardware benchmark script [#38672](https://github.com/ClickHouse/ClickHouse/pull/38672) ([Filatenkov Artur](https://github.com/FArthur-cmd)).
* Fix strange backport titles issues [#38679](https://github.com/ClickHouse/ClickHouse/pull/38679) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Follow-up to [#38568](https://github.com/ClickHouse/ClickHouse/issues/38568) [#38680](https://github.com/ClickHouse/ClickHouse/pull/38680) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix possible deadlocks with `MergeTreeData::Transaction` [#38702](https://github.com/ClickHouse/ClickHouse/pull/38702) ([alesapin](https://github.com/alesapin)).
* Fix backports diff [#38703](https://github.com/ClickHouse/ClickHouse/pull/38703) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Fix FillingTransform [#38705](https://github.com/ClickHouse/ClickHouse/pull/38705) ([Nikita Taranov](https://github.com/nickitat)).
* Try to improve backward compatibility check [#38717](https://github.com/ClickHouse/ClickHouse/pull/38717) ([Kruglov Pavel](https://github.com/Avogar)).
* SQL create drop index fix formatting [#38720](https://github.com/ClickHouse/ClickHouse/pull/38720) ([Maksim Kita](https://github.com/kitaisreal)).
* Provide sort description for output stream in ReadFromMergeTree step [#38721](https://github.com/ClickHouse/ClickHouse/pull/38721) ([Igor Nikonov](https://github.com/devcrafter)).
* Add exp_internal for expect tests [#38728](https://github.com/ClickHouse/ClickHouse/pull/38728) ([Azat Khuzhin](https://github.com/azat)).
* Fix CLICKHOUSE_TMP in tests (fixes broken CI) [#38733](https://github.com/ClickHouse/ClickHouse/pull/38733) ([Azat Khuzhin](https://github.com/azat)).
* Add SimpleCheck [#38744](https://github.com/ClickHouse/ClickHouse/pull/38744) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Tiny tests cleanup [#38749](https://github.com/ClickHouse/ClickHouse/pull/38749) ([Azat Khuzhin](https://github.com/azat)).
* Fix replication after improper merge process [#38752](https://github.com/ClickHouse/ClickHouse/pull/38752) ([Raúl Marín](https://github.com/Algunenano)).
* tests: make aggregate_state_exception_memory_leak deterministic [#38754](https://github.com/ClickHouse/ClickHouse/pull/38754) ([Azat Khuzhin](https://github.com/azat)).
* Bump jemalloc to fix possible assertion [#38757](https://github.com/ClickHouse/ClickHouse/pull/38757) ([Azat Khuzhin](https://github.com/azat)).
* Reintroduce nonNegativeDerivative() [#38774](https://github.com/ClickHouse/ClickHouse/pull/38774) ([Andrey Zvonov](https://github.com/zvonand)).
* Temporarily disable 01710_projection_fetch_long in BC check [#38798](https://github.com/ClickHouse/ClickHouse/pull/38798) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Use native Map type for OpenTelemetry attributes [#38814](https://github.com/ClickHouse/ClickHouse/pull/38814) ([Ilya Yatsishin](https://github.com/qoega)).
* Add test for segfault in Map combinator [#38831](https://github.com/ClickHouse/ClickHouse/pull/38831) ([Kruglov Pavel](https://github.com/Avogar)).
* Update libprotobuf-mutator + fix build [#38834](https://github.com/ClickHouse/ClickHouse/pull/38834) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
* Get files changed in master since release is branched [#38836](https://github.com/ClickHouse/ClickHouse/pull/38836) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* update integration tests doc [#38837](https://github.com/ClickHouse/ClickHouse/pull/38837) ([Bharat Nallan](https://github.com/bharatnc)).
* Revert of revert of smallish devguide update [#38850](https://github.com/ClickHouse/ClickHouse/pull/38850) ([Robert Schulze](https://github.com/rschu1ze)).
* Do not override compiler if it had been already set [#38856](https://github.com/ClickHouse/ClickHouse/pull/38856) ([Azat Khuzhin](https://github.com/azat)).
* Move check for denied allocations [#38858](https://github.com/ClickHouse/ClickHouse/pull/38858) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Refactoring of code around object storages, added LocalObjectStorage (extracted this diff from PR [#36171](https://github.com/ClickHouse/ClickHouse/issues/36171)) [#38860](https://github.com/ClickHouse/ClickHouse/pull/38860) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Backup Improvements 9 [#38861](https://github.com/ClickHouse/ClickHouse/pull/38861) ([Vitaly Baranov](https://github.com/vitlibar)).
* Simple cleanup: interpreters and parsers [#38876](https://github.com/ClickHouse/ClickHouse/pull/38876) ([Igor Nikonov](https://github.com/devcrafter)).
* Remove unnecessary log [#38892](https://github.com/ClickHouse/ClickHouse/pull/38892) ([Raúl Marín](https://github.com/Algunenano)).
* Update version_date.tsv and changelogs after v22.6.3.35-stable [#38894](https://github.com/ClickHouse/ClickHouse/pull/38894) ([github-actions[bot]](https://github.com/apps/github-actions)).
* Retry docker buildx commands with progressive sleep in between [#38898](https://github.com/ClickHouse/ClickHouse/pull/38898) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Avoid false positive red sanitizer asserts check in stress test [#38901](https://github.com/ClickHouse/ClickHouse/pull/38901) ([Kruglov Pavel](https://github.com/Avogar)).
* Interpreter cleanup: ContextPtr -> const ContextPtr & in parameters [#38902](https://github.com/ClickHouse/ClickHouse/pull/38902) ([Igor Nikonov](https://github.com/devcrafter)).
* Add a test for simdjson [#38933](https://github.com/ClickHouse/ClickHouse/pull/38933) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Fix a typo [#38938](https://github.com/ClickHouse/ClickHouse/pull/38938) ([Nikolay Degterinsky](https://github.com/evillique)).
* Avoid redundant join block transformation during planning. [#38943](https://github.com/ClickHouse/ClickHouse/pull/38943) ([Amos Bird](https://github.com/amosbird)).
* Rename NUMBER_OF_DIMENSIONS_MISMATHED const to NUMBER_OF_DIMENSIONS_MISMATCHED [#38947](https://github.com/ClickHouse/ClickHouse/pull/38947) ([Vladimir Galunshchikov](https://github.com/soyayaos)).
* More careful destructor in BackupImpl [#38949](https://github.com/ClickHouse/ClickHouse/pull/38949) ([Vitaly Baranov](https://github.com/vitlibar)).
* Avoid weird exception in Keeper [#38963](https://github.com/ClickHouse/ClickHouse/pull/38963) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Update version_date.tsv after v22.3.8.39-lts [#38969](https://github.com/ClickHouse/ClickHouse/pull/38969) ([github-actions[bot]](https://github.com/apps/github-actions)).
* Remove tag no-backward-compatibility-check for specific versions [#38971](https://github.com/ClickHouse/ClickHouse/pull/38971) ([Kruglov Pavel](https://github.com/Avogar)).
* add Hetzner benchmark [#38974](https://github.com/ClickHouse/ClickHouse/pull/38974) ([Tyler Hannan](https://github.com/tylerhannan)).
* Update version_date.tsv after v22.4.6.53-stable [#38975](https://github.com/ClickHouse/ClickHouse/pull/38975) ([github-actions[bot]](https://github.com/apps/github-actions)).
* Disable instrumentation of sanitizer death callback [#38977](https://github.com/ClickHouse/ClickHouse/pull/38977) ([Alexander Tokmakov](https://github.com/tavplubix)).
* add ryzen 9 5950 benchmark [#38979](https://github.com/ClickHouse/ClickHouse/pull/38979) ([Tyler Hannan](https://github.com/tylerhannan)).
* EXPLAIN AST rewrite: rename to optimize [#38980](https://github.com/ClickHouse/ClickHouse/pull/38980) ([Igor Nikonov](https://github.com/devcrafter)).
* add macbook pro core i7 2014 benchmark [#38981](https://github.com/ClickHouse/ClickHouse/pull/38981) ([Tyler Hannan](https://github.com/tylerhannan)).
* add Huawei TaiShan 920 Benchmark [#38982](https://github.com/ClickHouse/ClickHouse/pull/38982) ([Tyler Hannan](https://github.com/tylerhannan)).
* tests: unique ZooKeeper path for Replicated.*MergeTree tables [#38999](https://github.com/ClickHouse/ClickHouse/pull/38999) ([Azat Khuzhin](https://github.com/azat)).
* Try another suppression for [#38629](https://github.com/ClickHouse/ClickHouse/issues/38629) [#39009](https://github.com/ClickHouse/ClickHouse/pull/39009) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Add docker_server.py running to backport and release CIs [#39011](https://github.com/ClickHouse/ClickHouse/pull/39011) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Fix flaky `test_system_merges/test.py::test_mutation_simple` [#39013](https://github.com/ClickHouse/ClickHouse/pull/39013) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix assertion in full soring merge join [#39014](https://github.com/ClickHouse/ClickHouse/pull/39014) ([Vladimir C](https://github.com/vdimir)).
* Fix flaky 00620_optimize_on_nonleader_replica_zookeeper [#39019](https://github.com/ClickHouse/ClickHouse/pull/39019) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Suppress [#38643](https://github.com/ClickHouse/ClickHouse/issues/38643) [#39024](https://github.com/ClickHouse/ClickHouse/pull/39024) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Update url.md [#39025](https://github.com/ClickHouse/ClickHouse/pull/39025) ([Ilya Yatsishin](https://github.com/qoega)).
* Fix 'Tried to lock part ... for removal second time' [#39036](https://github.com/ClickHouse/ClickHouse/pull/39036) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Add more settings for randomization [#39039](https://github.com/ClickHouse/ClickHouse/pull/39039) ([Anton Popov](https://github.com/CurtizJ)).
* add ScaleFlux CSD3000 Benchmark [#39040](https://github.com/ClickHouse/ClickHouse/pull/39040) ([Tyler Hannan](https://github.com/tylerhannan)).
* BACKUP/RESTORE ON CLUSTER use async mode on replicas now. [#39046](https://github.com/ClickHouse/ClickHouse/pull/39046) ([Vitaly Baranov](https://github.com/vitlibar)).
* More stable `test_s3_zero_copy_ttl`, weakened requirement to move data to S3 in 0-5 seconds [#39064](https://github.com/ClickHouse/ClickHouse/pull/39064) ([Vladimir Chebotaryov](https://github.com/quickhouse)).
* Parameter --decompressor added to utils/self-extracting-executable/compressor [#39065](https://github.com/ClickHouse/ClickHouse/pull/39065) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Follow-up to [#39036](https://github.com/ClickHouse/ClickHouse/issues/39036) [#39091](https://github.com/ClickHouse/ClickHouse/pull/39091) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Update registerDiskS3.cpp [#39092](https://github.com/ClickHouse/ClickHouse/pull/39092) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix code in getLeastSupertype function [#39101](https://github.com/ClickHouse/ClickHouse/pull/39101) ([Kruglov Pavel](https://github.com/Avogar)).
* Remove some debug logging [#39102](https://github.com/ClickHouse/ClickHouse/pull/39102) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Prefix overridden add_executable() command with "clickhouse_" [#39108](https://github.com/ClickHouse/ClickHouse/pull/39108) ([Robert Schulze](https://github.com/rschu1ze)).
* Fix meilisearch tests [#39110](https://github.com/ClickHouse/ClickHouse/pull/39110) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Properly remove projection from part in case it was removed from table metadata. [#39119](https://github.com/ClickHouse/ClickHouse/pull/39119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Update cluster.py [#39120](https://github.com/ClickHouse/ClickHouse/pull/39120) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Tiny updates for tests. [#39127](https://github.com/ClickHouse/ClickHouse/pull/39127) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix data race in CompletedPipelineExecutor. [#39132](https://github.com/ClickHouse/ClickHouse/pull/39132) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix style again [#39133](https://github.com/ClickHouse/ClickHouse/pull/39133) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix path retrieval for Keeper's state [#39148](https://github.com/ClickHouse/ClickHouse/pull/39148) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
* Slightly better interface of waitForMutation [#39154](https://github.com/ClickHouse/ClickHouse/pull/39154) ([Amos Bird](https://github.com/amosbird)).
* ThreadPool fixes [#39160](https://github.com/ClickHouse/ClickHouse/pull/39160) ([Azat Khuzhin](https://github.com/azat)).
* Add test for [#39132](https://github.com/ClickHouse/ClickHouse/issues/39132) [#39173](https://github.com/ClickHouse/ClickHouse/pull/39173) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Suppression for BC check (`Cannot parse string 'Hello' as UInt64`) [#39176](https://github.com/ClickHouse/ClickHouse/pull/39176) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix 01961_roaring_memory_tracking test [#39187](https://github.com/ClickHouse/ClickHouse/pull/39187) ([Dmitry Novik](https://github.com/novikd)).
* Cleanup: done during [#38719](https://github.com/ClickHouse/ClickHouse/issues/38719) (SortingStep: deduce way to sort based on … [#39191](https://github.com/ClickHouse/ClickHouse/pull/39191) ([Igor Nikonov](https://github.com/devcrafter)).
* Fix exception in AsynchronousMetrics for s390x [#39193](https://github.com/ClickHouse/ClickHouse/pull/39193) ([Harry Lee](https://github.com/HarryLeeIBM)).
* Optimize accesses to system.stack_trace (filter by name before sending signal) [#39212](https://github.com/ClickHouse/ClickHouse/pull/39212) ([Azat Khuzhin](https://github.com/azat)).
* Enable warning "-Wdeprecated-dynamic-exception-spec" [#39213](https://github.com/ClickHouse/ClickHouse/pull/39213) ([Robert Schulze](https://github.com/rschu1ze)).
* Remove specialization global lock/unlock from ActionLocksManager [#39215](https://github.com/ClickHouse/ClickHouse/pull/39215) ([Azat Khuzhin](https://github.com/azat)).
* Turn some warnings on [#39223](https://github.com/ClickHouse/ClickHouse/pull/39223) ([Robert Schulze](https://github.com/rschu1ze)).
* Pass const std::string_view by value, not by reference [#39224](https://github.com/ClickHouse/ClickHouse/pull/39224) ([Kruglov Pavel](https://github.com/Avogar)).
* Minor fix for BC check [#39231](https://github.com/ClickHouse/ClickHouse/pull/39231) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Backport script [#39235](https://github.com/ClickHouse/ClickHouse/pull/39235) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Do not color logs on client if they are redirected to file [#39243](https://github.com/ClickHouse/ClickHouse/pull/39243) ([Anton Popov](https://github.com/CurtizJ)).
* Remove incorrect assertion [#39245](https://github.com/ClickHouse/ClickHouse/pull/39245) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Add X86 prefix to x86 performance tests [#39251](https://github.com/ClickHouse/ClickHouse/pull/39251) ([Robert Schulze](https://github.com/rschu1ze)).
* Check that the destination for a backup is not in use. [#39254](https://github.com/ClickHouse/ClickHouse/pull/39254) ([Vitaly Baranov](https://github.com/vitlibar)).
* Fix stacktraces in gdb in BC check [#39256](https://github.com/ClickHouse/ClickHouse/pull/39256) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Disable flaky test `test_s3_zero_copy_on_hybrid_storage` [#39258](https://github.com/ClickHouse/ClickHouse/pull/39258) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Enabled Wc99-extensions + Wsign-conversion [#39261](https://github.com/ClickHouse/ClickHouse/pull/39261) ([Robert Schulze](https://github.com/rschu1ze)).
* Pass const StringRef by value, not by reference [#39262](https://github.com/ClickHouse/ClickHouse/pull/39262) ([Robert Schulze](https://github.com/rschu1ze)).
* Fix assertion in transactions [#39263](https://github.com/ClickHouse/ClickHouse/pull/39263) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix macosx compilation due to endian.h [#39265](https://github.com/ClickHouse/ClickHouse/pull/39265) ([Jordi Villar](https://github.com/jrdi)).
* Another supression for BC check [#39276](https://github.com/ClickHouse/ClickHouse/pull/39276) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix builder image for releases w/o diagnostics tool [#39281](https://github.com/ClickHouse/ClickHouse/pull/39281) ([Azat Khuzhin](https://github.com/azat)).
* [RFC] Remove superior atomic from MergeTreeBackgroundExecutor and annotations for TSA [#39285](https://github.com/ClickHouse/ClickHouse/pull/39285) ([Azat Khuzhin](https://github.com/azat)).
* Fix clang tidy [#39288](https://github.com/ClickHouse/ClickHouse/pull/39288) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix running cmake with predefined cache (for development only) [#39295](https://github.com/ClickHouse/ClickHouse/pull/39295) ([Azat Khuzhin](https://github.com/azat)).
* Fix googletest contrib compilation (due to GTEST_HAS_POSIX_RE=0) [#39298](https://github.com/ClickHouse/ClickHouse/pull/39298) ([Azat Khuzhin](https://github.com/azat)).
* First try at reducing the use of StringRef [#39300](https://github.com/ClickHouse/ClickHouse/pull/39300) ([Robert Schulze](https://github.com/rschu1ze)).
* Whitespaces [#39303](https://github.com/ClickHouse/ClickHouse/pull/39303) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Add test for window function inside CASE [#39305](https://github.com/ClickHouse/ClickHouse/pull/39305) ([Dmitry Novik](https://github.com/novikd)).
* Simple Check should be updated on rerun [#39307](https://github.com/ClickHouse/ClickHouse/pull/39307) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Fix leaking of logger in clickhouse-disks [#39314](https://github.com/ClickHouse/ClickHouse/pull/39314) ([Azat Khuzhin](https://github.com/azat)).
* Update exception message [#39315](https://github.com/ClickHouse/ClickHouse/pull/39315) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix build clang-13 [#39318](https://github.com/ClickHouse/ClickHouse/pull/39318) ([alesapin](https://github.com/alesapin)).
* Auto set test name in integration tests [#39322](https://github.com/ClickHouse/ClickHouse/pull/39322) ([Vitaly Baranov](https://github.com/vitlibar)).
* Try fix flaky test_store_cleanup [#39334](https://github.com/ClickHouse/ClickHouse/pull/39334) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Do not start on unexpected Ordinary metadata [#39337](https://github.com/ClickHouse/ClickHouse/pull/39337) ([Alexander Tokmakov](https://github.com/tavplubix)).
* switch from mkdocs to Docusaurus [#39338](https://github.com/ClickHouse/ClickHouse/pull/39338) ([Dan Roscigno](https://github.com/DanRoscigno)).
* Fix flaky 01174_select_insert_isolation [#39339](https://github.com/ClickHouse/ClickHouse/pull/39339) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Better exception messages in schema inference [#39340](https://github.com/ClickHouse/ClickHouse/pull/39340) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix memory exceptions with transactions [#39341](https://github.com/ClickHouse/ClickHouse/pull/39341) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix typo [#39360](https://github.com/ClickHouse/ClickHouse/pull/39360) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix typo [#39361](https://github.com/ClickHouse/ClickHouse/pull/39361) ([Kruglov Pavel](https://github.com/Avogar)).
* Do not enqueue uneeded parts for check [#39366](https://github.com/ClickHouse/ClickHouse/pull/39366) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Avoid loading toolchain file multiple times to avoid confusing ccache [#39387](https://github.com/ClickHouse/ClickHouse/pull/39387) ([Azat Khuzhin](https://github.com/azat)).
* Fix make clean (due to crosscompile of llvm) [#39392](https://github.com/ClickHouse/ClickHouse/pull/39392) ([Azat Khuzhin](https://github.com/azat)).
* Disable real-time digest in Keeper by default [#39393](https://github.com/ClickHouse/ClickHouse/pull/39393) ([Antonio Andelic](https://github.com/antonio2368)).
* Fix interactive client with older server [#39413](https://github.com/ClickHouse/ClickHouse/pull/39413) ([Vitaly Baranov](https://github.com/vitlibar)).
* Fix BC check [#39414](https://github.com/ClickHouse/ClickHouse/pull/39414) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix flaky test test_login_as_dropped_user_xml. [#39415](https://github.com/ClickHouse/ClickHouse/pull/39415) ([Vitaly Baranov](https://github.com/vitlibar)).
* Introduce a dependency to libuv when building NATS [#39427](https://github.com/ClickHouse/ClickHouse/pull/39427) ([ltrk2](https://github.com/ltrk2)).
* Set default value cross_to_inner_join_rewrite = 1 [#39443](https://github.com/ClickHouse/ClickHouse/pull/39443) ([Vladimir C](https://github.com/vdimir)).
* Respect table alias for additional_table_filters. [#39456](https://github.com/ClickHouse/ClickHouse/pull/39456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
#### Performance optimization and Bug Fix
* Enabled `pread_threadpool` read method by default. It will increase read performance. Bug fix: if direct IO is enabled and the number of threads is large and `pread_threadpool` is used, it may cause a logical error. [#33653](https://github.com/ClickHouse/ClickHouse/pull/33653) ([Alexey Milovidov](https://github.com/alexey-milovidov)).

View File

@ -75,7 +75,7 @@ This will create the `programs/clickhouse` executable, which can be used with `c
The build requires the following components: The build requires the following components:
- Git (is used only to checkout the sources, its not needed for the build) - Git (is used only to checkout the sources, its not needed for the build)
- CMake 3.14 or newer - CMake 3.15 or newer
- Ninja - Ninja
- C++ compiler: clang-14 or newer - C++ compiler: clang-14 or newer
- Linker: lld - Linker: lld

View File

@ -194,18 +194,25 @@ Differs from the `TabSeparated` format in that the column names are written in t
During parsing, the first row is expected to contain the column names. You can use column names to determine their position and to check their correctness. During parsing, the first row is expected to contain the column names. You can use column names to determine their position and to check their correctness.
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1, If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1. the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped. Otherwise, the first row will be skipped.
:::
This format is also available under the name `TSVWithNames`. This format is also available under the name `TSVWithNames`.
## TabSeparatedWithNamesAndTypes {#tabseparatedwithnamesandtypes} ## TabSeparatedWithNamesAndTypes {#tabseparatedwithnamesandtypes}
Differs from the `TabSeparated` format in that the column names are written to the first row, while the column types are in the second row. Differs from the `TabSeparated` format in that the column names are written to the first row, while the column types are in the second row.
The first row with names is processed the same way as in `TabSeparatedWithNames` format.
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
If setting [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header) is set to 1, If setting [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header) is set to 1,
the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped. the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
:::
This format is also available under the name `TSVWithNamesAndTypes`. This format is also available under the name `TSVWithNamesAndTypes`.
@ -451,10 +458,24 @@ The CSV format supports the output of totals and extremes the same way as `TabSe
Also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). Also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames).
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
:::
## CSVWithNamesAndTypes {#csvwithnamesandtypes} ## CSVWithNamesAndTypes {#csvwithnamesandtypes}
Also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). Also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes).
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
If setting [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header) is set to 1,
the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
:::
## CustomSeparated {#format-customseparated} ## CustomSeparated {#format-customseparated}
Similar to [Template](#format-template), but it prints or reads all names and types of columns and uses escaping rule from [format_custom_escaping_rule](../operations/settings/settings.md#format_custom_escaping_rule) setting and delimiters from [format_custom_field_delimiter](../operations/settings/settings.md#format_custom_field_delimiter), [format_custom_row_before_delimiter](../operations/settings/settings.md#format_custom_row_before_delimiter), [format_custom_row_after_delimiter](../operations/settings/settings.md#format_custom_row_after_delimiter), [format_custom_row_between_delimiter](../operations/settings/settings.md#format_custom_row_between_delimiter), [format_custom_result_before_delimiter](../operations/settings/settings.md#format_custom_result_before_delimiter) and [format_custom_result_after_delimiter](../operations/settings/settings.md#format_custom_result_after_delimiter) settings, not from format strings. Similar to [Template](#format-template), but it prints or reads all names and types of columns and uses escaping rule from [format_custom_escaping_rule](../operations/settings/settings.md#format_custom_escaping_rule) setting and delimiters from [format_custom_field_delimiter](../operations/settings/settings.md#format_custom_field_delimiter), [format_custom_row_before_delimiter](../operations/settings/settings.md#format_custom_row_before_delimiter), [format_custom_row_after_delimiter](../operations/settings/settings.md#format_custom_row_after_delimiter), [format_custom_row_between_delimiter](../operations/settings/settings.md#format_custom_row_between_delimiter), [format_custom_result_before_delimiter](../operations/settings/settings.md#format_custom_result_before_delimiter) and [format_custom_result_after_delimiter](../operations/settings/settings.md#format_custom_result_after_delimiter) settings, not from format strings.
@ -465,10 +486,24 @@ There is also `CustomSeparatedIgnoreSpaces` format, which is similar to [Templat
Also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). Also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames).
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
:::
## CustomSeparatedWithNamesAndTypes {#customseparatedwithnamesandtypes} ## CustomSeparatedWithNamesAndTypes {#customseparatedwithnamesandtypes}
Also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). Also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes).
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
If setting [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header) is set to 1,
the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
:::
## SQLInsert {#sqlinsert} ## SQLInsert {#sqlinsert}
Outputs data as a sequence of `INSERT INTO table (columns...) VALUES (...), (...) ...;` statements. Outputs data as a sequence of `INSERT INTO table (columns...) VALUES (...), (...) ...;` statements.
@ -911,18 +946,46 @@ Differs from `JSONEachRow`/`JSONStringsEachRow` in that ClickHouse will also yie
Differs from `JSONCompactEachRow` format in that it also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). Differs from `JSONCompactEachRow` format in that it also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames).
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
:::
## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes} ## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes}
Differs from `JSONCompactEachRow` format in that it also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). Differs from `JSONCompactEachRow` format in that it also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes).
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
If setting [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header) is set to 1,
the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
:::
## JSONCompactStringsEachRowWithNames {#jsoncompactstringseachrowwithnames} ## JSONCompactStringsEachRowWithNames {#jsoncompactstringseachrowwithnames}
Differs from `JSONCompactStringsEachRow` in that in that it also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). Differs from `JSONCompactStringsEachRow` in that in that it also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames).
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
:::
## JSONCompactStringsEachRowWithNamesAndTypes {#jsoncompactstringseachrowwithnamesandtypes} ## JSONCompactStringsEachRowWithNamesAndTypes {#jsoncompactstringseachrowwithnamesandtypes}
Differs from `JSONCompactStringsEachRow` in that it also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). Differs from `JSONCompactStringsEachRow` in that it also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes).
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
If setting [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header) is set to 1,
the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
:::
```json ```json
["num", "str", "arr"] ["num", "str", "arr"]
["Int32", "String", "Array(UInt8)"] ["Int32", "String", "Array(UInt8)"]
@ -1199,6 +1262,12 @@ Similar to [RowBinary](#rowbinary), but with added header:
- [LEB128](https://en.wikipedia.org/wiki/LEB128)-encoded number of columns (N) - [LEB128](https://en.wikipedia.org/wiki/LEB128)-encoded number of columns (N)
- N `String`s specifying column names - N `String`s specifying column names
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
:::
## RowBinaryWithNamesAndTypes {#rowbinarywithnamesandtypes} ## RowBinaryWithNamesAndTypes {#rowbinarywithnamesandtypes}
Similar to [RowBinary](#rowbinary), but with added header: Similar to [RowBinary](#rowbinary), but with added header:
@ -1207,6 +1276,14 @@ Similar to [RowBinary](#rowbinary), but with added header:
- N `String`s specifying column names - N `String`s specifying column names
- N `String`s specifying column types - N `String`s specifying column types
:::warning
If setting [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header) is set to 1,
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields) is set to 1.
Otherwise, the first row will be skipped.
If setting [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header) is set to 1,
the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
:::
## Values {#data-format-values} ## Values {#data-format-values}
Prints every row in brackets. Rows are separated by commas. There is no comma after the last row. The values inside the brackets are also comma-separated. Numbers are output in a decimal format without quotes. Arrays are output in square brackets. Strings, dates, and dates with times are output in quotes. Escaping rules and parsing are similar to the [TabSeparated](#tabseparated) format. During formatting, extra spaces arent inserted, but during parsing, they are allowed and skipped (except for spaces inside array values, which are not allowed). [NULL](../sql-reference/syntax.md) is represented as `NULL`. Prints every row in brackets. Rows are separated by commas. There is no comma after the last row. The values inside the brackets are also comma-separated. Numbers are output in a decimal format without quotes. Arrays are output in square brackets. Strings, dates, and dates with times are output in quotes. Escaping rules and parsing are similar to the [TabSeparated](#tabseparated) format. During formatting, extra spaces arent inserted, but during parsing, they are allowed and skipped (except for spaces inside array values, which are not allowed). [NULL](../sql-reference/syntax.md) is represented as `NULL`.

View File

@ -45,7 +45,7 @@ Configuration template:
- `min_part_size` The minimum size of a data part. - `min_part_size` The minimum size of a data part.
- `min_part_size_ratio` The ratio of the data part size to the table size. - `min_part_size_ratio` The ratio of the data part size to the table size.
- `method` Compression method. Acceptable values: `lz4`, `lz4hc`, `zstd`. - `method` Compression method. Acceptable values: `lz4`, `lz4hc`, `zstd`,`deflate_qpl`.
- `level` Compression level. See [Codecs](../../sql-reference/statements/create/table.md#create-query-general-purpose-codecs). - `level` Compression level. See [Codecs](../../sql-reference/statements/create/table.md#create-query-general-purpose-codecs).
You can configure multiple `<case>` sections. You can configure multiple `<case>` sections.

View File

@ -3329,6 +3329,15 @@ Read more about [memory overcommit](memory-overcommit.md).
Default value: `1GiB`. Default value: `1GiB`.
## compatibility {#compatibility}
This setting changes other settings according to provided ClickHouse version.
If a behaviour in ClickHouse was changed by using a different default value for some setting, this compatibility setting allows you to use default values from previous versions for all the settings that were not set by the user.
This setting takes ClickHouse version number as a string, like `21.3`, `21.8`. Empty value means that this setting is disabled.
Disabled by default.
# Format settings {#format-settings} # Format settings {#format-settings}
## input_format_skip_unknown_fields {#input_format_skip_unknown_fields} ## input_format_skip_unknown_fields {#input_format_skip_unknown_fields}

View File

@ -836,7 +836,7 @@ Result:
## now ## now
Returns the current date and time. Returns the current date and time at the moment of query analysis. The function is a constant expression.
**Syntax** **Syntax**
@ -884,14 +884,20 @@ Result:
└──────────────────────┘ └──────────────────────┘
``` ```
## nowInBlock
Returns the current date and time at the moment of processing of each block of data. In contrast to the function `now`, it is not a constant expression, and the returned value will be different in different blocks for long-running queries.
It makes sense to use this function to generate the current time in long-running INSERT SELECT queries.
## today ## today
Accepts zero arguments and returns the current date at one of the moments of request execution. Accepts zero arguments and returns the current date at one of the moments of query analysis.
The same as toDate(now()). The same as toDate(now()).
## yesterday ## yesterday
Accepts zero arguments and returns yesterdays date at one of the moments of request execution. Accepts zero arguments and returns yesterdays date at one of the moments of query analysis.
The same as today() - 1. The same as today() - 1.
## timeSlot ## timeSlot

View File

@ -494,22 +494,21 @@ If the s string is non-empty and does not contain the c character at
Returns the string s that was converted from the encoding in from to the encoding in to. Returns the string s that was converted from the encoding in from to the encoding in to.
## Base58Encode(plaintext[, alphabet_name]), Base58Decode(encoded_text[, alphabet_name]) ## Base58Encode(plaintext), Base58Decode(encoded_text)
Accepts a String and encodes/decodes it using [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) encoding scheme using specified alphabet. Accepts a String and encodes/decodes it using [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) encoding scheme using "Bitcoin" alphabet.
**Syntax** **Syntax**
```sql ```sql
base58Encode(decoded[, alphabet_name]) base58Encode(decoded)
base58Decode(encoded[, alphabet_name]) base58Decode(encoded)
``` ```
**Arguments** **Arguments**
- `decoded` — [String](../../sql-reference/data-types/string.md) column or constant. - `decoded` — [String](../../sql-reference/data-types/string.md) column or constant.
- `encoded` — [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid base58-encoded value, an exception is thrown. - `encoded` — [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid base58-encoded value, an exception is thrown.
- `alphabet_name` — String constant. Specifies alphabet used for encoding. Possible values: `gmp`, `bitcoin`, `ripple`, `flickr`. Default: `bitcoin`.
**Returned value** **Returned value**
@ -522,17 +521,17 @@ Type: [String](../../sql-reference/data-types/string.md).
Query: Query:
``` sql ``` sql
SELECT base58Encode('encode', 'flickr'); SELECT base58Encode('Encoded');
SELECT base58Decode('izCFiDUY', 'ripple'); SELECT base58Encode('3dc8KtHrwM');
``` ```
Result: Result:
```text ```text
┌─base58Encode('encode', 'flickr')─┐ ┌─encodeBase58('Encoded')─┐
SvyTHb1D 3dc8KtHrwM
└──────────────────────────────────┘ └──────────────────────────────────┘
┌─base58Decode('izCFiDUY', 'ripple')─┐ ┌─decodeBase58('3dc8KtHrwM')─┐
decode Encoded
└────────────────────────────────────┘ └────────────────────────────────────┘
``` ```

View File

@ -96,7 +96,7 @@ For more information, see the link: [RE2](https://github.com/google/re2/blob/mas
## translate(s, from, to) ## translate(s, from, to)
The function replaces characters in the string s in accordance with one-to-one character mapping defined by from and to strings. from and to must be ASCII strings of the same size. Non-ASCII characters in the original string are not modified. The function replaces characters in the string s in accordance with one-to-one character mapping defined by from and to strings. from and to must be constant ASCII strings of the same size. Non-ASCII characters in the original string are not modified.
Example: Example:
@ -112,7 +112,7 @@ SELECT translate('Hello, World!', 'delor', 'DELOR') AS res
## translateUTF8(string, from, to) ## translateUTF8(string, from, to)
Similar to previous function, but works with UTF-8 arguments. from and to must be valid UTF-8 strings of the same size. Similar to previous function, but works with UTF-8 arguments. from and to must be valid constant UTF-8 strings of the same size.
Example: Example:

View File

@ -248,6 +248,13 @@ ClickHouse supports general purpose codecs and specialized codecs.
High compression levels are useful for asymmetric scenarios, like compress once, decompress repeatedly. Higher levels mean better compression and higher CPU usage. High compression levels are useful for asymmetric scenarios, like compress once, decompress repeatedly. Higher levels mean better compression and higher CPU usage.
#### DEFLATE_QPL
`DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library, which has dependency on Intel Hardware:
- DEFLATE_QPL is only supported on systems with AVX2/AVX512/IAA.
- DEFLATE_QPL-compressed data can only be transferred between nodes with AVX2/AVX512/IAA.
### Specialized Codecs ### Specialized Codecs
These codecs are designed to make compression more effective by using specific features of data. Some of these codecs do not compress data themself. Instead, they prepare the data for a common purpose codec, which compresses it better than without this preparation. These codecs are designed to make compression more effective by using specific features of data. Some of these codecs do not compress data themself. Instead, they prepare the data for a common purpose codec, which compresses it better than without this preparation.

View File

@ -18,7 +18,6 @@ sidebar_label: "Используемые сторонние библиотеки
| aws-c-common | [Apache](https://github.com/ClickHouse-Extras/aws-c-common/blob/736a82d1697c108b04a277e66438a7f4e19b6857/LICENSE) | | aws-c-common | [Apache](https://github.com/ClickHouse-Extras/aws-c-common/blob/736a82d1697c108b04a277e66438a7f4e19b6857/LICENSE) |
| aws-c-event-stream | [Apache](https://github.com/ClickHouse-Extras/aws-c-event-stream/blob/3bc33662f9ccff4f4cbcf9509cc78c26e022fde0/LICENSE) | | aws-c-event-stream | [Apache](https://github.com/ClickHouse-Extras/aws-c-event-stream/blob/3bc33662f9ccff4f4cbcf9509cc78c26e022fde0/LICENSE) |
| aws-checksums | [Apache](https://github.com/ClickHouse-Extras/aws-checksums/blob/519d6d9093819b6cf89ffff589a27ef8f83d0f65/LICENSE) | | aws-checksums | [Apache](https://github.com/ClickHouse-Extras/aws-checksums/blob/519d6d9093819b6cf89ffff589a27ef8f83d0f65/LICENSE) |
| base58 | [MIT](https://github.com/ClickHouse/base-x/blob/3e58874643c087f57e82b0ff03825c933fab945a/LICENSE) |
| base64 | [BSD 2-clause](https://github.com/ClickHouse-Extras/Turbo-Base64/blob/af9b331f2b4f30b41c70f3a571ff904a8251c1d3/LICENSE) | | base64 | [BSD 2-clause](https://github.com/ClickHouse-Extras/Turbo-Base64/blob/af9b331f2b4f30b41c70f3a571ff904a8251c1d3/LICENSE) |
| boost | [Boost](https://github.com/ClickHouse-Extras/boost/blob/9cf09dbfd55a5c6202dedbdf40781a51b02c2675/LICENSE_1_0.txt) | | boost | [Boost](https://github.com/ClickHouse-Extras/boost/blob/9cf09dbfd55a5c6202dedbdf40781a51b02c2675/LICENSE_1_0.txt) |
| boringssl | [BSD](https://github.com/ClickHouse-Extras/boringssl/blob/a6a2e2ab3e44d97ce98e51c558e989f211de7eb3/LICENSE) | | boringssl | [BSD](https://github.com/ClickHouse-Extras/boringssl/blob/a6a2e2ab3e44d97ce98e51c558e989f211de7eb3/LICENSE) |

View File

@ -15,16 +15,15 @@ $ make
Генерация данных: Генерация данных:
:::danger "Внимание" :::warning "Внимание"
-s 100 dbgen генерирует 600 миллионов строк (67 ГБ) -s 100 dbgen генерирует 600 миллионов строк (67 ГБ)
-s 1000 dbgen генерирует 6 миллиардов строк (занимает много времени) -s 1000 dbgen генерирует 6 миллиардов строк (занимает много времени)
:::
``` bash ``` bash
$ ./dbgen -s 1000 -T c $ ./dbgen -s 1000 -T c
$ ./dbgen -s 1000 -T l $ ./dbgen -s 1000 -T l
$ ./dbgen -s 1000 -T p $ ./dbgen -s 1000 -T p
$ ./dbgen -s 1000 -T s
$ ./dbgen -s 1000 -T d
``` ```
Создание таблиц в Кликхауз: Создание таблиц в Кликхауз:
@ -105,11 +104,10 @@ $ clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" < lineorder.tbl
``` sql ``` sql
SET max_memory_usage = 20000000000; SET max_memory_usage = 20000000000;
CREATE TABLE lineorder_flat CREATE TABLE lineorder_flat
ENGINE = MergeTree ENGINE = MergeTree ORDER BY (LO_ORDERDATE, LO_ORDERKEY)
PARTITION BY toYear(LO_ORDERDATE) AS SELECT
ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS
SELECT
l.LO_ORDERKEY AS LO_ORDERKEY, l.LO_ORDERKEY AS LO_ORDERKEY,
l.LO_LINENUMBER AS LO_LINENUMBER, l.LO_LINENUMBER AS LO_LINENUMBER,
l.LO_CUSTKEY AS LO_CUSTKEY, l.LO_CUSTKEY AS LO_CUSTKEY,

View File

@ -19,6 +19,7 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT
| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | | [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ |
| [CSV](#csv) | ✔ | ✔ | | [CSV](#csv) | ✔ | ✔ |
| [CSVWithNames](#csvwithnames) | ✔ | ✔ | | [CSVWithNames](#csvwithnames) | ✔ | ✔ |
| [CSVWithNamesAndTypes](#csvwithnamesandtypes) | ✔ | ✔ |
| [CustomSeparated](#format-customseparated) | ✔ | ✔ | | [CustomSeparated](#format-customseparated) | ✔ | ✔ |
| [CustomSeparatedWithNames](#customseparatedwithnames) | ✔ | ✔ | | [CustomSeparatedWithNames](#customseparatedwithnames) | ✔ | ✔ |
| [CustomSeparatedWithNamesAndTypes](#customseparatedwithnamesandtypes) | ✔ | ✔ | | [CustomSeparatedWithNamesAndTypes](#customseparatedwithnamesandtypes) | ✔ | ✔ |
@ -52,6 +53,7 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT
| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | | [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ |
| [ORC](#data-format-orc) | ✔ | ✔ | | [ORC](#data-format-orc) | ✔ | ✔ |
| [RowBinary](#rowbinary) | ✔ | ✔ | | [RowBinary](#rowbinary) | ✔ | ✔ |
| [RowBinaryWithNames](#rowbinarywithnames) | ✔ | ✔ |
| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | | [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ |
| [Native](#native) | ✔ | ✔ | | [Native](#native) | ✔ | ✔ |
| [Null](#null) | ✗ | ✔ | | [Null](#null) | ✗ | ✔ |
@ -171,6 +173,12 @@ SELECT * FROM nestedt FORMAT TSV
При парсинге первая строка должна содержать имена столбцов. Вы можете использовать имена столбцов, чтобы указать их порядок расположения, или чтобы проверить их корректность. При парсинге первая строка должна содержать имена столбцов. Вы можете использовать имена столбцов, чтобы указать их порядок расположения, или чтобы проверить их корректность.
:::warning
Если включен параметр [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header),
столбцы из входных данных будут сопоставлены со столбцами таблицы по их именам, столбцы с неизвестными именами будут пропущены, если включен параметр [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields).
В противном случае первая строка будет пропущена.
:::
Этот формат также доступен под именем `TSVWithNames`. Этот формат также доступен под именем `TSVWithNames`.
## TabSeparatedWithNamesAndTypes {#tabseparatedwithnamesandtypes} ## TabSeparatedWithNamesAndTypes {#tabseparatedwithnamesandtypes}
@ -178,6 +186,14 @@ SELECT * FROM nestedt FORMAT TSV
Отличается от формата `TabSeparated` тем, что в первой строке пишутся имена столбцов, а во второй - типы столбцов. Отличается от формата `TabSeparated` тем, что в первой строке пишутся имена столбцов, а во второй - типы столбцов.
При парсинге, первая и вторая строка полностью игнорируется. При парсинге, первая и вторая строка полностью игнорируется.
:::warning
Если включен параметр [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header),
столбцы из входных данных будут сопоставлены со столбцами таблицы по их именам, столбцы с неизвестными именами будут пропущены, если включен параметр [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields).
В противном случае первая строка будет пропущена.
Если включен параметр [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header),
типы из входных данных будут сравниваться с типами соответствующих столбцов из таблицы. В противном случае вторая строка будет пропущена.
:::
Этот формат также доступен под именем `TSVWithNamesAndTypes`. Этот формат также доступен под именем `TSVWithNamesAndTypes`.
## Template {#format-template} ## Template {#format-template}
@ -374,6 +390,24 @@ $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FOR
Выводит также заголовок, аналогично [TabSeparatedWithNames](#tabseparatedwithnames). Выводит также заголовок, аналогично [TabSeparatedWithNames](#tabseparatedwithnames).
:::warning
Если включен параметр [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header),
столбцы из входных данных будут сопоставлены со столбцами таблицы по их именам, столбцы с неизвестными именами будут пропущены, если включен параметр [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields).
В противном случае первая строка будет пропущена.
:::
## CSVWithNamesAndTypes {#csvwithnamesandtypes}
В первой строке пишутся имена столбцов, а во второй - типы столбцов, аналогично [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes)
:::warning
Если включен параметр [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header),
столбцы из входных данных будут сопоставлены со столбцами таблицы по их именам, столбцы с неизвестными именами будут пропущены, если включен параметр [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields).
В противном случае первая строка будет пропущена.
Если включен параметр [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header),
типы из входных данных будут сравниваться с типами соответствующих столбцов из таблицы. В противном случае вторая строка будет пропущена.
:::
## CustomSeparated {#format-customseparated} ## CustomSeparated {#format-customseparated}
Аналогичен [Template](#format-template), но выводит (или считывает) все имена и типы столбцов, используя для них правило экранирования из настройки [format_custom_escaping_rule](../operations/settings/settings.md#format-custom-escaping-rule) и разделители из настроек [format_custom_field_delimiter](../operations/settings/settings.md#format-custom-field-delimiter), [format_custom_row_before_delimiter](../operations/settings/settings.md#format-custom-row-before-delimiter), [format_custom_row_after_delimiter](../operations/settings/settings.md#format-custom-row-after-delimiter), [format_custom_row_between_delimiter](../operations/settings/settings.md#format-custom-row-between-delimiter), [format_custom_result_before_delimiter](../operations/settings/settings.md#format-custom-result-before-delimiter) и [format_custom_result_after_delimiter](../operations/settings/settings.md#format-custom-result-after-delimiter), а не из форматных строк. Аналогичен [Template](#format-template), но выводит (или считывает) все имена и типы столбцов, используя для них правило экранирования из настройки [format_custom_escaping_rule](../operations/settings/settings.md#format-custom-escaping-rule) и разделители из настроек [format_custom_field_delimiter](../operations/settings/settings.md#format-custom-field-delimiter), [format_custom_row_before_delimiter](../operations/settings/settings.md#format-custom-row-before-delimiter), [format_custom_row_after_delimiter](../operations/settings/settings.md#format-custom-row-after-delimiter), [format_custom_row_between_delimiter](../operations/settings/settings.md#format-custom-row-between-delimiter), [format_custom_result_before_delimiter](../operations/settings/settings.md#format-custom-result-before-delimiter) и [format_custom_result_after_delimiter](../operations/settings/settings.md#format-custom-result-after-delimiter), а не из форматных строк.
@ -384,10 +418,24 @@ $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FOR
Выводит также заголовок с именами столбцов, аналогичен формату [TabSeparatedWithNames](#tabseparatedwithnames). Выводит также заголовок с именами столбцов, аналогичен формату [TabSeparatedWithNames](#tabseparatedwithnames).
:::warning
Если включен параметр [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header),
столбцы из входных данных будут сопоставлены со столбцами таблицы по их именам, столбцы с неизвестными именами будут пропущены, если включен параметр [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields).
В противном случае первая строка будет пропущена.
:::
## CustomSeparatedWithNamesAndTypes {#customseparatedwithnamesandtypes} ## CustomSeparatedWithNamesAndTypes {#customseparatedwithnamesandtypes}
Выводит также два заголовка с именами и типами столбцов, аналогичен формату [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). Выводит также два заголовка с именами и типами столбцов, аналогичен формату [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes).
:::warning
Если включен параметр [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header),
столбцы из входных данных будут сопоставлены со столбцами таблицы по их именам, столбцы с неизвестными именами будут пропущены, если включен параметр [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields).
В противном случае первая строка будет пропущена.
Если включен параметр [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header),
типы из входных данных будут сравниваться с типами соответствующих столбцов из таблицы. В противном случае вторая строка будет пропущена.
:::
## JSON {#json} ## JSON {#json}
Выводит данные в формате JSON. Кроме таблицы с данными, также выводятся имена и типы столбцов, и некоторая дополнительная информация - общее количество выведенных строк, а также количество строк, которое могло бы быть выведено, если бы не было LIMIT-а. Пример: Выводит данные в формате JSON. Кроме таблицы с данными, также выводятся имена и типы столбцов, и некоторая дополнительная информация - общее количество выведенных строк, а также количество строк, которое могло бы быть выведено, если бы не было LIMIT-а. Пример:
@ -660,6 +708,14 @@ SELECT * FROM json_square_brackets;
Отличается от `JSONCompactEachRow`/`JSONCompactStringsEachRow` тем, что имена и типы столбцов записываются как первые две строки. Отличается от `JSONCompactEachRow`/`JSONCompactStringsEachRow` тем, что имена и типы столбцов записываются как первые две строки.
:::warning
Если включен параметр [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header),
столбцы из входных данных будут сопоставлены со столбцами таблицы по их именам, столбцы с неизвестными именами будут пропущены, если включен параметр [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields).
В противном случае первая строка будет пропущена.
Если включен параметр [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header),
типы из входных данных будут сравниваться с типами соответствующих столбцов из таблицы. В противном случае вторая строка будет пропущена.
:::
```json ```json
["'hello'", "multiply(42, number)", "range(5)"] ["'hello'", "multiply(42, number)", "range(5)"]
["String", "UInt64", "Array(UInt8)"] ["String", "UInt64", "Array(UInt8)"]
@ -904,6 +960,20 @@ Array представлены как длина в формате varint (unsig
Для поддержки [NULL](../sql-reference/syntax.md#null-literal) перед каждым значением типа [Nullable](../sql-reference/data-types/nullable.md) следует байт содержащий 1 или 0. Если байт 1, то значение равно NULL, и этот байт интерпретируется как отдельное значение (т.е. после него следует значение следующего поля). Если байт 0, то после байта следует значение поля (не равно NULL). Для поддержки [NULL](../sql-reference/syntax.md#null-literal) перед каждым значением типа [Nullable](../sql-reference/data-types/nullable.md) следует байт содержащий 1 или 0. Если байт 1, то значение равно NULL, и этот байт интерпретируется как отдельное значение (т.е. после него следует значение следующего поля). Если байт 0, то после байта следует значение поля (не равно NULL).
## RowBinaryWithNames {#rowbinarywithnames}
То же самое что [RowBinary](#rowbinary), но добавляется заголовок:
- Количество колонок - N, закодированное [LEB128](https://en.wikipedia.org/wiki/LEB128),
- N строк (`String`) с именами колонок,
:::warning
Если включен параметр [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header),
столбцы из входных данных будут сопоставлены со столбцами таблицы по их именам, столбцы с неизвестными именами будут пропущены, если включен параметр [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields).
В противном случае первая строка будет пропущена.
:::
## RowBinaryWithNamesAndTypes {#rowbinarywithnamesandtypes} ## RowBinaryWithNamesAndTypes {#rowbinarywithnamesandtypes}
То же самое что [RowBinary](#rowbinary), но добавляется заголовок: То же самое что [RowBinary](#rowbinary), но добавляется заголовок:
@ -912,6 +982,14 @@ Array представлены как длина в формате varint (unsig
- N строк (`String`) с именами колонок, - N строк (`String`) с именами колонок,
- N строк (`String`) с типами колонок. - N строк (`String`) с типами колонок.
:::warning
Если включен параметр [input_format_with_names_use_header](../operations/settings/settings.md#input_format_with_names_use_header),
столбцы из входных данных будут сопоставлены со столбцами таблицы по их именам, столбцы с неизвестными именами будут пропущены, если включен параметр [input_format_skip_unknown_fields](../operations/settings/settings.md#input_format_skip_unknown_fields).
В противном случае первая строка будет пропущена.
Если включен параметр [input_format_with_types_use_header](../operations/settings/settings.md#input_format_with_types_use_header),
типы из входных данных будут сравниваться с типами соответствующих столбцов из таблицы. В противном случае вторая строка будет пропущена.
:::
## Values {#data-format-values} ## Values {#data-format-values}
Выводит каждую строку в скобках. Строки разделены запятыми. После последней строки запятой нет. Значения внутри скобок также разделены запятыми. Числа выводятся в десятичном виде без кавычек. Массивы выводятся в квадратных скобках. Строки, даты, даты-с-временем выводятся в кавычках. Правила экранирования и особенности парсинга аналогичны формату [TabSeparated](#tabseparated). При форматировании, лишние пробелы не ставятся, а при парсинге - допустимы и пропускаются (за исключением пробелов внутри значений типа массив, которые недопустимы). [NULL](../sql-reference/syntax.md) представляется как `NULL`. Выводит каждую строку в скобках. Строки разделены запятыми. После последней строки запятой нет. Значения внутри скобок также разделены запятыми. Числа выводятся в десятичном виде без кавычек. Массивы выводятся в квадратных скобках. Строки, даты, даты-с-временем выводятся в кавычках. Правила экранирования и особенности парсинга аналогичны формату [TabSeparated](#tabseparated). При форматировании, лишние пробелы не ставятся, а при парсинге - допустимы и пропускаются (за исключением пробелов внутри значений типа массив, которые недопустимы). [NULL](../sql-reference/syntax.md) представляется как `NULL`.

View File

@ -44,7 +44,7 @@ ClickHouse перезагружает встроенные словари с з
- `min_part_size` - Минимальный размер части таблицы. - `min_part_size` - Минимальный размер части таблицы.
- `min_part_size_ratio` - Отношение размера минимальной части таблицы к полному размеру таблицы. - `min_part_size_ratio` - Отношение размера минимальной части таблицы к полному размеру таблицы.
- `method` - Метод сжатия. Возможные значения: `lz4`, `lz4hc`, `zstd`. - `method` - Метод сжатия. Возможные значения: `lz4`, `lz4hc`, `zstd`,`deflate_qpl`.
- `level` Уровень сжатия. См. [Кодеки](../../sql-reference/statements/create/table/#create-query-common-purpose-codecs). - `level` Уровень сжатия. См. [Кодеки](../../sql-reference/statements/create/table/#create-query-common-purpose-codecs).
Можно сконфигурировать несколько разделов `<case>`. Можно сконфигурировать несколько разделов `<case>`.

View File

@ -527,7 +527,7 @@ SELECT * FROM table_with_enum_column_for_tsv_insert;
- [Использование вложенных структур](../../interfaces/formats.md#jsoneachrow-nested) with the `JSONEachRow` format. - [Использование вложенных структур](../../interfaces/formats.md#jsoneachrow-nested) with the `JSONEachRow` format.
## input_format_with_names_use_header {#settings-input-format-with-names-use-header} ## input_format_with_names_use_header {#input_format_with_names_use_header}
Включает или отключает проверку порядка столбцов при вставке данных. Включает или отключает проверку порядка столбцов при вставке данных.
@ -535,8 +535,38 @@ SELECT * FROM table_with_enum_column_for_tsv_insert;
Поддерживаемые форматы: Поддерживаемые форматы:
- [CSVWithNames](../../interfaces/formats.md#csvwithnames) - [CSVWithNames](../../interfaces/formats.md#csvwithnames)
- [TabSeparatedWithNames](../../interfaces/formats.md#tabseparatedwithnames) - [CSVWithNamesAndTypes](../../interfaces/formats.md#csvwithnamesandtypes)
- [TabSeparatedWithNames](../../interfaces/formats.md#tabseparatedwithnames)
- [TabSeparatedWithNamesAndTypes](../../interfaces/formats.md#tabseparatedwithnamesandtypes)
- [JSONCompactEachRowWithNames](../../interfaces/formats.md#jsoncompacteachrowwithnames)
- [JSONCompactEachRowWithNamesAndTypes](../../interfaces/formats.md#jsoncompacteachrowwithnamesandtypes)
- [JSONCompactStringsEachRowWithNames](../../interfaces/formats.md#jsoncompactstringseachrowwithnames)
- [JSONCompactStringsEachRowWithNamesAndTypes](../../interfaces/formats.md#jsoncompactstringseachrowwithnamesandtypes)
- [RowBinaryWithNames](../../interfaces/formats.md#rowbinarywithnames)
- [RowBinaryWithNamesAndTypes](../../interfaces/formats.md#rowbinarywithnamesandtypes)
- [CustomSeparatedWithNames](../../interfaces/formats.md#customseparatedwithnames)
- [CustomSeparatedWithNamesAndTypes](../../interfaces/formats.md#customseparatedwithnamesandtypes)
Возможные значения:
- 0 — выключена.
- 1 — включена.
Значение по умолчанию: 1.
## input_format_with_types_use_header {#input_format_with_types_use_header}
Определяет, должен ли синтаксический анализатор формата проверять, соответствуют ли типы данных из входных данных типам данных из целевой таблицы.
Поддерживаемые форматы:
- [CSVWithNamesAndTypes](../../interfaces/formats.md#csvwithnamesandtypes)
- [TabSeparatedWithNamesAndTypes](../../interfaces/formats.md#tabseparatedwithnamesandtypes)
- [JSONCompactEachRowWithNamesAndTypes](../../interfaces/formats.md#jsoncompacteachrowwithnamesandtypes)
- [JSONCompactStringsEachRowWithNamesAndTypes](../../interfaces/formats.md#jsoncompactstringseachrowwithnamesandtypes)
- [RowBinaryWithNamesAndTypes](../../interfaces/formats.md#rowbinarywithnamesandtypes-rowbinarywithnamesandtypes)
- [CustomSeparatedWithNamesAndTypes](../../interfaces/formats.md#customseparatedwithnamesandtypes)
Возможные значения: Возможные значения:

View File

@ -490,22 +490,21 @@ SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY (key1, key2);
Возвращает сконвертированную из кодировки from в кодировку to строку s. Возвращает сконвертированную из кодировки from в кодировку to строку s.
## Base58Encode(plaintext[, alphabet_name]), Base58Decode(plaintext[, alphabet_name]) {#base58} ## Base58Encode(plaintext), Base58Decode(encoded_text) {#base58}
Принимает на вход строку или колонку строк и кодирует/раскодирует их с помощью схемы кодирования [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) с использованием указанного алфавита. Принимает на вход строку или колонку строк и кодирует/раскодирует их с помощью схемы кодирования [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) с использованием стандартного алфавита Bitcoin.
**Синтаксис** **Синтаксис**
```sql ```sql
base58Encode(decoded[, alphabet_name]) encodeBase58(decoded)
base58Decode(encoded[, alphabet_name]) decodeBase58(encoded)
``` ```
**Аргументы** **Аргументы**
- `decoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). - `decoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md).
- `encoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). Если входная строка не является корректным кодом для какой-либо другой строки, возникнет исключение `1001`. - `encoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). Если входная строка не является корректным кодом для какой-либо другой строки, возникнет исключение `1001`.
- `alphabet_name` — Строковая константа. Указывает алфавит, для которого необходимо получить код. Может принимать одно из следующих значений: `gmp`, `bitcoin`, `ripple`, `flickr`. По умолчанию: `bitcoin`.
**Возвращаемое значение** **Возвращаемое значение**
@ -518,16 +517,16 @@ base58Decode(encoded[, alphabet_name])
Запрос: Запрос:
``` sql ``` sql
SELECT base58Encode('encode', 'flickr'); SELECT encodeBase58('encode');
SELECT base58Decode('izCFiDUY', 'ripple'); SELECT decodeBase58('izCFiDUY');
``` ```
Результат: Результат:
```text ```text
┌─base58Encode('encode', 'flickr')─┐ ┌─encodeBase58('encode', 'flickr')─┐
│ SvyTHb1D │ │ SvyTHb1D │
└──────────────────────────────────┘ └──────────────────────────────────┘
┌─base58Decode('izCFiDUY', 'ripple')─┐ ┌─decodeBase58('izCFiDUY', 'ripple')─┐
│ decode │ │ decode │
└────────────────────────────────────┘ └────────────────────────────────────┘
``` ```

View File

@ -85,7 +85,7 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res
## translate(s, from, to) ## translate(s, from, to)
Данная функция заменяет символы в строке s в соответствии с поэлементным отображением определяемым строками from и to. from и to должны быть корректными ASCII строками одного размера. Не ASCII символы в оригинальной строке не изменяются. Данная функция заменяет символы в строке s в соответствии с поэлементным отображением определяемым строками from и to. from и to должны быть корректными константными ASCII строками одного размера. Не ASCII символы в оригинальной строке не изменяются.
Example: Example:
@ -101,7 +101,7 @@ SELECT translate('Hello, World!', 'delor', 'DELOR') AS res
## translateUTF8(string, from, to) ## translateUTF8(string, from, to)
Аналогично предыдущей функции, но работает со строками, состоящими из UTF-8 символов. from и to должны быть корректными UTF-8 строками одного размера. Аналогично предыдущей функции, но работает со строками, состоящими из UTF-8 символов. from и to должны быть корректными константными UTF-8 строками одного размера.
Example: Example:

View File

@ -1,50 +1,94 @@
## How ClickHouse documentation is generated? {#how-clickhouse-documentation-is-generated} ## Generating ClickHouse documentation {#how-clickhouse-documentation-is-generated}
ClickHouse documentation is built using [build.py](build.py) script that uses [mkdocs](https://www.mkdocs.org) library and its dependencies to separately build all version of documentations (all languages in either single and multi page mode) as static HTMLs for each single page version. The results are then put in the correct directory structure. It is recommended to use Python 3.7 to run this script. ClickHouse documentation is built using [Docusaurus](https://docusaurus.io).
[release.sh](release.sh) also pulls static files needed for [official ClickHouse website](https://clickhouse.com) from [../../website](../../website) folder then pushes to specified GitHub repo to be served via [GitHub Pages](https://pages.github.com). ## Check the look of your documentation changes {#how-to-check-if-the-documentation-will-look-fine}
## How to check if the documentation will look fine? {#how-to-check-if-the-documentation-will-look-fine} There are a few options that are all useful depending on how large or complex your edits are.
There are few options that are all useful depending on how large or complex your edits are. ### Use the GitHub web interface to edit
### Use GitHub web interface to edit Every page in the docs has an **Edit this page** link that opens the page in the GitHub editor. GitHub has Markdown support with a preview feature. The details of GitHub Markdown and the documentation Markdown are a bit different but generally this is close enough, and the person merging your PR will build the docs and check them.
GitHub has Markdown support with preview feature, but the details of GitHub Markdown dialect are a bit different in ClickHouse documentation. ### Install a Markdown editor or plugin for your IDE {#install-markdown-editor-or-plugin-for-your-ide}
### Install Markdown editor or plugin for your IDE {#install-markdown-editor-or-plugin-for-your-ide} Usually, these plugins provide a preview of how the markdown will render, and they catch basic errors like unclosed tags very early.
Usually those also have some way to preview how Markdown will look like, which allows to catch basic errors like unclosed tags very early.
### Use build.py {#use-build-py} ## Build the docs locally {#use-build-py}
Itll take some effort to go through, but the result will be very close to production documentation. You can build the docs locally. It takes a few minutes to set up, but once you have done it the first time, the process is very simple.
For the first time youll need to: ### Clone the repos
#### 1. Set up virtualenv The documentation is in two repos, clone both of them:
- [ClickHouse/ClickHouse](https://github.com/ClickHouse/ClickHouse)
- [ClickHouse/ClickHouse-docs](https://github.com/ClickHouse/clickhouse-docs)
``` bash ### Install Node.js
$ cd ClickHouse/docs/tools
$ mkdir venv The documentation is built with Docusaurus, which requires Node.js. We recommend version 16. Install [Node.js](https://nodejs.org/en/download/).
$ virtualenv -p $(which python3) venv
$ source venv/bin/activate ### Copy files into place
$ pip3 install -r requirements.txt
Docusaurus expects all of the markdown files to be located in the directory tree `clickhouse-docs/docs/`. This is not the way our repos are set up, so some copying of files is needed to build the docs:
```bash
# from the parent directory of both the ClickHouse/ClickHouse and ClickHouse-clickhouse-docs repos:
cp -r ClickHouse/docs/en/development clickhouse-docs/docs/en/
cp -r ClickHouse/docs/en/engines clickhouse-docs/docs/en/
cp -r ClickHouse/docs/en/getting-started clickhouse-docs/docs/en/
cp -r ClickHouse/docs/en/interfaces clickhouse-docs/docs/en/
cp -r ClickHouse/docs/en/operations clickhouse-docs/docs/en/
cp -r ClickHouse/docs/en/sql-reference clickhouse-docs/docs/en/
cp -r ClickHouse/docs/ru/* clickhouse-docs/docs/ru/
cp -r ClickHouse/docs/zh clickhouse-docs/docs/
``` ```
#### 2. Run build.py #### Note: Symlinks will not work.
### Setup Docusaurus
When all prerequisites are installed, running `build.py` without args (there are some, check `build.py --help`) will generate `ClickHouse/docs/build` folder with complete static html website. There are two commands that you may need to use with Docusaurus:
- `yarn install`
- `yarn start`
The easiest way to see the result is to use `--livereload=8888` argument of build.py. Alternatively, you can manually launch a HTTP server to serve the docs, for example by running `cd ClickHouse/docs/build && python3 -m http.server 8888`. Then go to http://localhost:8888 in browser. Feel free to use any other port instead of 8888. #### Install Docusaurus and its dependencies:
```bash
cd clickhouse-docs
yarn install
```
#### Start a development Docusaurus environment
This command will start Docusaurus in development mode, which means that as you edit source (for example, `.md` files) files the changes will be rendered into HTML files and served by the Docusaurus development server.
```bash
yarn start
```
### Make your changes to the markdown files
Edit your files. Remember that if you are editing files in the `ClickHouse/ClickHouse` repo then you should edit them
in that repo and then copy the edited file into the `ClickHouse/clickhouse-docs/` directory structure so that they are updated in your develoment environment.
`yarn start` probably opened a browser for you when you ran it; if not, open a browser to `http://localhost:3000/docs/en/intro` and navigate to the documentation that you are changing. If you have already made the changes, you can verify them here; if not, make them, and you will see the page update as you save the changes.
## How to change code highlighting? {#how-to-change-code-hl} ## How to change code highlighting? {#how-to-change-code-hl}
ClickHouse does not use mkdocs `highlightjs` feature. It uses modified pygments styles instead. Code highlighting is based on the language chosen for your code blocks. Specify the language when you start the code block:
If you want to change code highlighting, edit the `website/css/highlight.css` file.
Currently, an [eighties](https://github.com/idleberg/base16-pygments/blob/master/css/base16-eighties.dark.css) theme
is used.
<pre lang="no-highlight"><code>```sql
SELECT firstname from imdb.actors;
```
</code></pre>
```sql
SELECT firstname from imdb.actors;
```
If you need a language supported then open an issue in [ClickHouse-docs](https://github.com/ClickHouse/clickhouse-docs/issues).
## How to subscribe on documentation changes? {#how-to-subscribe-on-documentation-changes} ## How to subscribe on documentation changes? {#how-to-subscribe-on-documentation-changes}
At the moment theres no easy way to do just that, but you can consider: At the moment theres no easy way to do just that, but you can consider:

View File

@ -15,15 +15,14 @@ $ make
开始生成数据: 开始生成数据:
!!! warning "注意" :::warning "注意"
使用`-s 100`dbgen 将生成 6 亿行数据(67GB), 如果使用`-s 1000`它会生成 60 亿行数据(这需要很多时间)) 使用`-s 100`dbgen 将生成 6 亿行数据(67GB), 如果使用`-s 1000`它会生成 60 亿行数据(这需要很多时间))
:::
```bash ```bash
$ ./dbgen -s 1000 -T c $ ./dbgen -s 1000 -T c
$ ./dbgen -s 1000 -T l $ ./dbgen -s 1000 -T l
$ ./dbgen -s 1000 -T p $ ./dbgen -s 1000 -T p
$ ./dbgen -s 1000 -T s
$ ./dbgen -s 1000 -T d
``` ```
在 ClickHouse 中创建数据表: 在 ClickHouse 中创建数据表:
@ -106,10 +105,8 @@ $ clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" < lineorder.tbl
SET max_memory_usage = 20000000000; SET max_memory_usage = 20000000000;
CREATE TABLE lineorder_flat CREATE TABLE lineorder_flat
ENGINE = MergeTree ENGINE = MergeTree ORDER BY (LO_ORDERDATE, LO_ORDERKEY)
PARTITION BY toYear(LO_ORDERDATE) AS SELECT
ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS
SELECT
l.LO_ORDERKEY AS LO_ORDERKEY, l.LO_ORDERKEY AS LO_ORDERKEY,
l.LO_LINENUMBER AS LO_LINENUMBER, l.LO_LINENUMBER AS LO_LINENUMBER,
l.LO_CUSTKEY AS LO_CUSTKEY, l.LO_CUSTKEY AS LO_CUSTKEY,

View File

@ -18,7 +18,7 @@ option (ENABLE_CLICKHOUSE_SERVER "Server mode (main mode)" ${ENABLE_CLICKHOUSE_A
option (ENABLE_CLICKHOUSE_CLIENT "Client mode (interactive tui/shell that connects to the server)" option (ENABLE_CLICKHOUSE_CLIENT "Client mode (interactive tui/shell that connects to the server)"
${ENABLE_CLICKHOUSE_ALL}) ${ENABLE_CLICKHOUSE_ALL})
if (CLICKHOUSE_SPLIT_BINARY OR NOT ENABLE_UTILS) if (CLICKHOUSE_SPLIT_BINARY)
option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" OFF) option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" OFF)
else () else ()
option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" ON) option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" ON)
@ -434,6 +434,9 @@ else ()
endif () endif ()
set (CLICKHOUSE_BUNDLE) set (CLICKHOUSE_BUNDLE)
if (ENABLE_CLICKHOUSE_SELF_EXTRACTING)
list(APPEND CLICKHOUSE_BUNDLE self-extracting)
endif ()
if (ENABLE_CLICKHOUSE_SERVER) if (ENABLE_CLICKHOUSE_SERVER)
add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse) add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)

View File

@ -102,9 +102,34 @@ void Client::processError(const String & query) const
} }
void Client::showWarnings()
{
try
{
std::vector<String> messages = loadWarningMessages();
if (!messages.empty())
{
std::cout << "Warnings:" << std::endl;
for (const auto & message : messages)
std::cout << " * " << message << std::endl;
std::cout << std::endl;
}
}
catch (...)
{
/// Ignore exception
}
}
/// Make query to get all server warnings /// Make query to get all server warnings
std::vector<String> Client::loadWarningMessages() std::vector<String> Client::loadWarningMessages()
{ {
/// Older server versions cannot execute the query loading warnings.
constexpr UInt64 min_server_revision_to_load_warnings = DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED;
if (server_revision < min_server_revision_to_load_warnings)
return {};
std::vector<String> messages; std::vector<String> messages;
connection->sendQuery(connection_parameters.timeouts, connection->sendQuery(connection_parameters.timeouts,
"SELECT * FROM viewIfPermitted(SELECT message FROM system.warnings ELSE null('message String'))", "SELECT * FROM viewIfPermitted(SELECT message FROM system.warnings ELSE null('message String'))",
@ -226,25 +251,9 @@ try
connect(); connect();
/// Load Warnings at the beginning of connection /// Show warnings at the beginning of connection.
if (is_interactive && !config().has("no-warnings")) if (is_interactive && !config().has("no-warnings"))
{ showWarnings();
try
{
std::vector<String> messages = loadWarningMessages();
if (!messages.empty())
{
std::cout << "Warnings:" << std::endl;
for (const auto & message : messages)
std::cout << " * " << message << std::endl;
std::cout << std::endl;
}
}
catch (...)
{
/// Ignore exception
}
}
if (is_interactive && !delayed_interactive) if (is_interactive && !delayed_interactive)
{ {
@ -370,7 +379,7 @@ void Client::connect()
} }
server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_version_patch); server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_version_patch);
load_suggestions = is_interactive && (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false)); load_suggestions = is_interactive && (server_revision >= Suggest::MIN_SERVER_REVISION) && !config().getBool("disable_suggestion", false);
if (server_display_name = connection->getServerDisplayName(connection_parameters.timeouts); server_display_name.empty()) if (server_display_name = connection->getServerDisplayName(connection_parameters.timeouts); server_display_name.empty())
server_display_name = config().getString("host", "localhost"); server_display_name = config().getString("host", "localhost");

View File

@ -45,6 +45,7 @@ protected:
private: private:
void printChangedSettings() const; void printChangedSettings() const;
void showWarnings();
std::vector<String> loadWarningMessages(); std::vector<String> loadWarningMessages();
}; };
} }

View File

@ -79,6 +79,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
("block-size,b", po::value<unsigned>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") ("block-size,b", po::value<unsigned>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size")
("hc", "use LZ4HC instead of LZ4") ("hc", "use LZ4HC instead of LZ4")
("zstd", "use ZSTD instead of LZ4") ("zstd", "use ZSTD instead of LZ4")
("deflate_qpl", "use deflate_qpl instead of LZ4")
("codec", po::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4") ("codec", po::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4")
("level", po::value<int>(), "compression level for codecs specified via flags") ("level", po::value<int>(), "compression level for codecs specified via flags")
("none", "use no compression instead of LZ4") ("none", "use no compression instead of LZ4")
@ -103,6 +104,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
bool decompress = options.count("decompress"); bool decompress = options.count("decompress");
bool use_lz4hc = options.count("hc"); bool use_lz4hc = options.count("hc");
bool use_zstd = options.count("zstd"); bool use_zstd = options.count("zstd");
bool use_deflate_qpl = options.count("deflate_qpl");
bool stat_mode = options.count("stat"); bool stat_mode = options.count("stat");
bool use_none = options.count("none"); bool use_none = options.count("none");
unsigned block_size = options["block-size"].as<unsigned>(); unsigned block_size = options["block-size"].as<unsigned>();
@ -110,7 +112,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
if (options.count("codec")) if (options.count("codec"))
codecs = options["codec"].as<std::vector<std::string>>(); codecs = options["codec"].as<std::vector<std::string>>();
if ((use_lz4hc || use_zstd || use_none) && !codecs.empty()) if ((use_lz4hc || use_zstd || use_deflate_qpl || use_none) && !codecs.empty())
throw Exception("Wrong options, codec flags like --zstd and --codec options are mutually exclusive", ErrorCodes::BAD_ARGUMENTS); throw Exception("Wrong options, codec flags like --zstd and --codec options are mutually exclusive", ErrorCodes::BAD_ARGUMENTS);
if (!codecs.empty() && options.count("level")) if (!codecs.empty() && options.count("level"))
@ -122,6 +124,8 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
method_family = "LZ4HC"; method_family = "LZ4HC";
else if (use_zstd) else if (use_zstd)
method_family = "ZSTD"; method_family = "ZSTD";
else if (use_deflate_qpl)
method_family = "DEFLATE_QPL";
else if (use_none) else if (use_none)
method_family = "NONE"; method_family = "NONE";

View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "ICommand.h" #include "ICommand.h"
#include <Interpreters/Context.h>
namespace DB namespace DB
{ {

View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "ICommand.h" #include "ICommand.h"
#include <Interpreters/Context.h>
namespace DB namespace DB
{ {

View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "ICommand.h" #include "ICommand.h"
#include <Interpreters/Context.h>
namespace DB namespace DB
{ {

View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "ICommand.h" #include "ICommand.h"
#include <Interpreters/Context.h>
namespace DB namespace DB
{ {

View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "ICommand.h" #include "ICommand.h"
#include <Interpreters/Context.h>
namespace DB namespace DB
{ {

View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "ICommand.h" #include "ICommand.h"
#include <Interpreters/Context.h>
namespace DB namespace DB
{ {

View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "ICommand.h" #include "ICommand.h"
#include <Interpreters/Context.h>
namespace DB namespace DB
{ {

View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "ICommand.h" #include "ICommand.h"
#include <Interpreters/Context.h>
namespace DB namespace DB
{ {

View File

@ -1,6 +1,18 @@
if (NOT(
CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR
)
)
set (COMPRESSOR "${CMAKE_BINARY_DIR}/native/utils/self-extracting-executable/pre_compressor")
set (DECOMPRESSOR "--decompressor=${CMAKE_BINARY_DIR}/utils/self-extracting-executable/decompressor")
else ()
set (COMPRESSOR "${CMAKE_BINARY_DIR}/utils/self-extracting-executable/compressor")
endif ()
add_custom_target (self-extracting ALL add_custom_target (self-extracting ALL
${CMAKE_COMMAND} -E remove clickhouse ${CMAKE_COMMAND} -E remove clickhouse
COMMAND ${CMAKE_BINARY_DIR}/utils/self-extracting-executable/compressor clickhouse ../clickhouse COMMAND ${COMPRESSOR} ${DECOMPRESSOR} clickhouse ../clickhouse
DEPENDS clickhouse compressor DEPENDS clickhouse compressor
) )

View File

@ -110,18 +110,24 @@ namespace
} }
/// Returns the host name by its address. /// Returns the host name by its address.
String getHostByAddress(const IPAddress & address) Strings getHostsByAddress(const IPAddress & address)
{ {
String host = DNSResolver::instance().reverseResolve(address); auto hosts = DNSResolver::instance().reverseResolve(address);
/// Check that PTR record is resolved back to client address if (hosts.empty())
if (!isAddressOfHost(address, host)) throw Exception(ErrorCodes::DNS_ERROR, "{} could not be resolved", address.toString());
throw Exception("Host " + String(host) + " isn't resolved back to " + address.toString(), ErrorCodes::DNS_ERROR);
return host;
for (const auto & host : hosts)
{
/// Check that PTR record is resolved back to client address
if (!isAddressOfHost(address, host))
throw Exception(ErrorCodes::DNS_ERROR, "Host {} isn't resolved back to {}", host, address.toString());
}
return hosts;
} }
void parseLikePatternIfIPSubnet(const String & pattern, IPSubnet & subnet, IPAddress::Family address_family) void parseLikePatternIfIPSubnet(const String & pattern, IPSubnet & subnet, IPAddress::Family address_family)
{ {
size_t slash = pattern.find('/'); size_t slash = pattern.find('/');
@ -520,20 +526,29 @@ bool AllowedClientHosts::contains(const IPAddress & client_address) const
return true; return true;
/// Check `name_regexps`. /// Check `name_regexps`.
std::optional<String> resolved_host; std::optional<Strings> resolved_hosts;
auto check_name_regexp = [&](const String & name_regexp_) auto check_name_regexp = [&](const String & name_regexp_)
{ {
try try
{ {
if (boost::iequals(name_regexp_, "localhost")) if (boost::iequals(name_regexp_, "localhost"))
return is_client_local(); return is_client_local();
if (!resolved_host) if (!resolved_hosts)
resolved_host = getHostByAddress(client_v6); {
if (resolved_host->empty()) resolved_hosts = getHostsByAddress(client_address);
return false; }
Poco::RegularExpression re(name_regexp_);
Poco::RegularExpression::Match match; for (const auto & host : resolved_hosts.value())
return re.match(*resolved_host, match) != 0; {
Poco::RegularExpression re(name_regexp_);
Poco::RegularExpression::Match match;
if (re.match(host, match) != 0)
{
return true;
}
}
return false;
} }
catch (const Exception & e) catch (const Exception & e)
{ {

View File

@ -15,6 +15,7 @@
#include <Poco/JSON/Stringifier.h> #include <Poco/JSON/Stringifier.h>
#include <boost/algorithm/string/case_conv.hpp> #include <boost/algorithm/string/case_conv.hpp>
#include <boost/range/adaptor/map.hpp> #include <boost/range/adaptor/map.hpp>
#include <base/range.h>
#include <filesystem> #include <filesystem>
#include <fstream> #include <fstream>

View File

@ -509,7 +509,6 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
if (referrals) if (referrals)
{ {
SCOPE_EXIT({ SCOPE_EXIT({
// ldap_value_free(referrals);
ber_memvfree(reinterpret_cast<void **>(referrals)); ber_memvfree(reinterpret_cast<void **>(referrals));
referrals = nullptr; referrals = nullptr;
}); });

View File

@ -8,6 +8,8 @@
namespace DB namespace DB
{ {
struct Array;
Array getAggregateFunctionParametersArray( Array getAggregateFunctionParametersArray(
const ASTPtr & expression_list, const ASTPtr & expression_list,
const std::string & error_context, const std::string & error_context,

View File

@ -7,6 +7,7 @@
#include <IO/Archives/hasRegisteredArchiveFileExtension.h> #include <IO/Archives/hasRegisteredArchiveFileExtension.h>
#include <Poco/Util/AbstractConfiguration.h> #include <Poco/Util/AbstractConfiguration.h>
#include <filesystem> #include <filesystem>
#include <Interpreters/Context.h>
namespace DB namespace DB

View File

@ -346,6 +346,12 @@ set_source_files_properties(
Columns/ColumnString.cpp Columns/ColumnString.cpp
PROPERTIES COMPILE_FLAGS "${X86_INTRINSICS_FLAGS}") PROPERTIES COMPILE_FLAGS "${X86_INTRINSICS_FLAGS}")
if (ENABLE_QPL)
set_source_files_properties(
Compression/CompressionCodecDeflateQpl.cpp
PROPERTIES COMPILE_FLAGS "-mwaitpkg")
endif ()
target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2_st) target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2_st)
target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2) target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2)
@ -376,7 +382,7 @@ if (TARGET ch_contrib::rdkafka)
endif() endif()
if (TARGET ch_contrib::nats_io) if (TARGET ch_contrib::nats_io)
dbms_target_link_libraries(PRIVATE ch_contrib::nats_io) dbms_target_link_libraries(PRIVATE ch_contrib::nats_io ch_contrib::uv)
endif() endif()
if (TARGET ch_contrib::sasl2) if (TARGET ch_contrib::sasl2)
@ -447,6 +453,9 @@ if (TARGET ch_contrib::avrocpp)
dbms_target_link_libraries(PRIVATE ch_contrib::avrocpp) dbms_target_link_libraries(PRIVATE ch_contrib::avrocpp)
endif () endif ()
set_source_files_properties(Common/CaresPTRResolver.cpp PROPERTIES COMPILE_FLAGS -Wno-reserved-identifier)
target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::c-ares)
if (TARGET OpenSSL::Crypto) if (TARGET OpenSSL::Crypto)
dbms_target_link_libraries (PRIVATE OpenSSL::Crypto) dbms_target_link_libraries (PRIVATE OpenSSL::Crypto)
target_link_libraries (clickhouse_common_io PRIVATE OpenSSL::Crypto) target_link_libraries (clickhouse_common_io PRIVATE OpenSSL::Crypto)
@ -527,6 +536,10 @@ endif ()
target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::lz4) target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::lz4)
if (TARGET ch_contrib::qpl)
dbms_target_link_libraries(PUBLIC ch_contrib::qpl)
endif ()
dbms_target_link_libraries(PRIVATE _boost_context) dbms_target_link_libraries(PRIVATE _boost_context)
if (ENABLE_NLP) if (ENABLE_NLP)

View File

@ -152,7 +152,6 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src)
auto & dst_column_host_name = typeid_cast<ColumnString &>(*mutable_columns[name_pos["host_name"]]); auto & dst_column_host_name = typeid_cast<ColumnString &>(*mutable_columns[name_pos["host_name"]]);
auto & dst_array_current_time = typeid_cast<ColumnUInt32 &>(*mutable_columns[name_pos["current_time"]]).getData(); auto & dst_array_current_time = typeid_cast<ColumnUInt32 &>(*mutable_columns[name_pos["current_time"]]).getData();
// auto & dst_array_thread_id = typeid_cast<ColumnUInt64 &>(*mutable_columns[name_pos["thread_id"]]).getData();
auto & dst_array_type = typeid_cast<ColumnInt8 &>(*mutable_columns[name_pos["type"]]).getData(); auto & dst_array_type = typeid_cast<ColumnInt8 &>(*mutable_columns[name_pos["type"]]).getData();
auto & dst_column_name = typeid_cast<ColumnString &>(*mutable_columns[name_pos["name"]]); auto & dst_column_name = typeid_cast<ColumnString &>(*mutable_columns[name_pos["name"]]);
auto & dst_array_value = typeid_cast<ColumnInt64 &>(*mutable_columns[name_pos["value"]]).getData(); auto & dst_array_value = typeid_cast<ColumnInt64 &>(*mutable_columns[name_pos["value"]]).getData();

View File

@ -28,8 +28,8 @@ public:
template <typename ConnectionType> template <typename ConnectionType>
void load(ContextPtr context, const ConnectionParameters & connection_parameters, Int32 suggestion_limit); void load(ContextPtr context, const ConnectionParameters & connection_parameters, Int32 suggestion_limit);
/// Older server versions cannot execute the query above. /// Older server versions cannot execute the query loading suggestions.
static constexpr int MIN_SERVER_REVISION = 54406; static constexpr int MIN_SERVER_REVISION = DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED;
private: private:
void fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query); void fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query);

View File

@ -132,14 +132,12 @@ namespace
ColumnLowCardinality::ColumnLowCardinality(MutableColumnPtr && column_unique_, MutableColumnPtr && indexes_, bool is_shared) ColumnLowCardinality::ColumnLowCardinality(MutableColumnPtr && column_unique_, MutableColumnPtr && indexes_, bool is_shared)
: dictionary(std::move(column_unique_), is_shared), idx(std::move(indexes_)) : dictionary(std::move(column_unique_), is_shared), idx(std::move(indexes_))
{ {
// idx.check(getDictionary().size());
} }
void ColumnLowCardinality::insert(const Field & x) void ColumnLowCardinality::insert(const Field & x)
{ {
compactIfSharedDictionary(); compactIfSharedDictionary();
idx.insertPosition(dictionary.getColumnUnique().uniqueInsert(x)); idx.insertPosition(dictionary.getColumnUnique().uniqueInsert(x));
// idx.check(getDictionary().size());
} }
void ColumnLowCardinality::insertDefault() void ColumnLowCardinality::insertDefault()
@ -167,15 +165,12 @@ void ColumnLowCardinality::insertFrom(const IColumn & src, size_t n)
const auto & nested = *low_cardinality_src->getDictionary().getNestedColumn(); const auto & nested = *low_cardinality_src->getDictionary().getNestedColumn();
idx.insertPosition(dictionary.getColumnUnique().uniqueInsertFrom(nested, position)); idx.insertPosition(dictionary.getColumnUnique().uniqueInsertFrom(nested, position));
} }
// idx.check(getDictionary().size());
} }
void ColumnLowCardinality::insertFromFullColumn(const IColumn & src, size_t n) void ColumnLowCardinality::insertFromFullColumn(const IColumn & src, size_t n)
{ {
compactIfSharedDictionary(); compactIfSharedDictionary();
idx.insertPosition(dictionary.getColumnUnique().uniqueInsertFrom(src, n)); idx.insertPosition(dictionary.getColumnUnique().uniqueInsertFrom(src, n));
// idx.check(getDictionary().size());
} }
void ColumnLowCardinality::insertRangeFrom(const IColumn & src, size_t start, size_t length) void ColumnLowCardinality::insertRangeFrom(const IColumn & src, size_t start, size_t length)
@ -205,7 +200,6 @@ void ColumnLowCardinality::insertRangeFrom(const IColumn & src, size_t start, si
auto inserted_indexes = dictionary.getColumnUnique().uniqueInsertRangeFrom(*used_keys, 0, used_keys->size()); auto inserted_indexes = dictionary.getColumnUnique().uniqueInsertRangeFrom(*used_keys, 0, used_keys->size());
idx.insertPositionsRange(*inserted_indexes->index(*sub_idx, 0), 0, length); idx.insertPositionsRange(*inserted_indexes->index(*sub_idx, 0), 0, length);
} }
// idx.check(getDictionary().size());
} }
void ColumnLowCardinality::insertRangeFromFullColumn(const IColumn & src, size_t start, size_t length) void ColumnLowCardinality::insertRangeFromFullColumn(const IColumn & src, size_t start, size_t length)
@ -213,7 +207,6 @@ void ColumnLowCardinality::insertRangeFromFullColumn(const IColumn & src, size_t
compactIfSharedDictionary(); compactIfSharedDictionary();
auto inserted_indexes = dictionary.getColumnUnique().uniqueInsertRangeFrom(src, start, length); auto inserted_indexes = dictionary.getColumnUnique().uniqueInsertRangeFrom(src, start, length);
idx.insertPositionsRange(*inserted_indexes, 0, length); idx.insertPositionsRange(*inserted_indexes, 0, length);
// idx.check(getDictionary().size());
} }
static void checkPositionsAreLimited(const IColumn & positions, UInt64 limit) static void checkPositionsAreLimited(const IColumn & positions, UInt64 limit)
@ -254,14 +247,12 @@ void ColumnLowCardinality::insertRangeFromDictionaryEncodedColumn(const IColumn
compactIfSharedDictionary(); compactIfSharedDictionary();
auto inserted_indexes = dictionary.getColumnUnique().uniqueInsertRangeFrom(keys, 0, keys.size()); auto inserted_indexes = dictionary.getColumnUnique().uniqueInsertRangeFrom(keys, 0, keys.size());
idx.insertPositionsRange(*inserted_indexes->index(positions, 0), 0, positions.size()); idx.insertPositionsRange(*inserted_indexes->index(positions, 0), 0, positions.size());
// idx.check(getDictionary().size());
} }
void ColumnLowCardinality::insertData(const char * pos, size_t length) void ColumnLowCardinality::insertData(const char * pos, size_t length)
{ {
compactIfSharedDictionary(); compactIfSharedDictionary();
idx.insertPosition(dictionary.getColumnUnique().uniqueInsertData(pos, length)); idx.insertPosition(dictionary.getColumnUnique().uniqueInsertData(pos, length));
// idx.check(getDictionary().size());
} }
StringRef ColumnLowCardinality::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const StringRef ColumnLowCardinality::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const
@ -276,7 +267,6 @@ const char * ColumnLowCardinality::deserializeAndInsertFromArena(const char * po
const char * new_pos; const char * new_pos;
idx.insertPosition(dictionary.getColumnUnique().uniqueDeserializeAndInsertFromArena(pos, new_pos)); idx.insertPosition(dictionary.getColumnUnique().uniqueDeserializeAndInsertFromArena(pos, new_pos));
// idx.check(getDictionary().size());
return new_pos; return new_pos;
} }

View File

@ -273,14 +273,6 @@ llvm::Value * ColumnNullable::compileComparator(llvm::IRBuilderBase & builder, l
b.CreateCondBr(lhs_or_rhs_are_null, lhs_or_rhs_are_null_block, lhs_rhs_are_not_null_block); b.CreateCondBr(lhs_or_rhs_are_null, lhs_or_rhs_are_null_block, lhs_rhs_are_not_null_block);
// if (unlikely(lval_is_null || rval_is_null))
// {
// if (lval_is_null && rval_is_null)
// return 0;
// else
// return lval_is_null ? null_direction_hint : -null_direction_hint;
// }
b.SetInsertPoint(lhs_or_rhs_are_null_block); b.SetInsertPoint(lhs_or_rhs_are_null_block);
auto * lhs_equals_rhs_result = llvm::ConstantInt::getSigned(b.getInt8Ty(), 0); auto * lhs_equals_rhs_result = llvm::ConstantInt::getSigned(b.getInt8Ty(), 0);
llvm::Value * lhs_and_rhs_are_null = b.CreateAnd(lhs_is_null_value, rhs_is_null_value); llvm::Value * lhs_and_rhs_are_null = b.CreateAnd(lhs_is_null_value, rhs_is_null_value);
@ -288,8 +280,6 @@ llvm::Value * ColumnNullable::compileComparator(llvm::IRBuilderBase & builder, l
llvm::Value * lhs_or_rhs_are_null_block_result = b.CreateSelect(lhs_and_rhs_are_null, lhs_equals_rhs_result, lhs_is_null_result); llvm::Value * lhs_or_rhs_are_null_block_result = b.CreateSelect(lhs_and_rhs_are_null, lhs_equals_rhs_result, lhs_is_null_result);
b.CreateBr(join_block); b.CreateBr(join_block);
// getNestedColumn().compareAt(n, m, nested_rhs, null_direction_hint);
b.SetInsertPoint(lhs_rhs_are_not_null_block); b.SetInsertPoint(lhs_rhs_are_not_null_block);
llvm::Value * lhs_rhs_are_not_null_block_result llvm::Value * lhs_rhs_are_not_null_block_result
= nested_column->compileComparator(builder, lhs_unwrapped_value, rhs_unwrapped_value, nan_direction_hint); = nested_column->compileComparator(builder, lhs_unwrapped_value, rhs_unwrapped_value, nan_direction_hint);

View File

@ -548,7 +548,6 @@ MutableColumnPtr ColumnUnique<ColumnType>::uniqueInsertRangeImpl(
} }
} }
// checkIndexes(*positions_column, column->size() + (overflowed_keys ? overflowed_keys->size() : 0));
return std::move(positions_column); return std::move(positions_column);
} }

View File

@ -514,8 +514,6 @@ private:
return allocateFromFreeRegion(*free_region, size); return allocateFromFreeRegion(*free_region, size);
} }
// std::cerr << "Requested size: " << size << "\n";
/// Evict something from cache and continue. /// Evict something from cache and continue.
while (true) while (true)
{ {

View File

@ -0,0 +1,109 @@
#include "CaresPTRResolver.h"
#include <arpa/inet.h>
#include <sys/select.h>
#include <Common/Exception.h>
#include "ares.h"
#include "netdb.h"
namespace DB
{
namespace ErrorCodes
{
extern const int DNS_ERROR;
}
static void callback(void * arg, int status, int, struct hostent * host)
{
auto * ptr_records = reinterpret_cast<std::vector<std::string>*>(arg);
if (status == ARES_SUCCESS && host->h_aliases)
{
int i = 0;
while (auto * ptr_record = host->h_aliases[i])
{
ptr_records->emplace_back(ptr_record);
i++;
}
}
}
CaresPTRResolver::CaresPTRResolver(CaresPTRResolver::provider_token) : channel(nullptr)
{
/*
* ares_library_init is not thread safe. Currently, the only other usage of c-ares seems to be in grpc.
* In grpc, ares_library_init seems to be called only in Windows.
* See https://github.com/grpc/grpc/blob/master/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc#L1187
* That means it's safe to init it here, but we should be cautious when introducing new code that depends on c-ares and even updates
* to grpc. As discussed in https://github.com/ClickHouse/ClickHouse/pull/37827#discussion_r919189085, c-ares should be adapted to be atomic
* */
if (ares_library_init(ARES_LIB_INIT_ALL) != ARES_SUCCESS || ares_init(&channel) != ARES_SUCCESS)
{
throw DB::Exception("Failed to initialize c-ares", DB::ErrorCodes::DNS_ERROR);
}
}
CaresPTRResolver::~CaresPTRResolver()
{
ares_destroy(channel);
ares_library_cleanup();
}
std::vector<std::string> CaresPTRResolver::resolve(const std::string & ip)
{
std::vector<std::string> ptr_records;
resolve(ip, ptr_records);
wait();
return ptr_records;
}
std::vector<std::string> CaresPTRResolver::resolve_v6(const std::string & ip)
{
std::vector<std::string> ptr_records;
resolve_v6(ip, ptr_records);
wait();
return ptr_records;
}
void CaresPTRResolver::resolve(const std::string & ip, std::vector<std::string> & response)
{
in_addr addr;
inet_pton(AF_INET, ip.c_str(), &addr);
ares_gethostbyaddr(channel, reinterpret_cast<const void*>(&addr), sizeof(addr), AF_INET, callback, &response);
}
void CaresPTRResolver::resolve_v6(const std::string & ip, std::vector<std::string> & response)
{
in6_addr addr;
inet_pton(AF_INET6, ip.c_str(), &addr);
ares_gethostbyaddr(channel, reinterpret_cast<const void*>(&addr), sizeof(addr), AF_INET6, callback, &response);
}
void CaresPTRResolver::wait()
{
timeval * tvp, tv;
fd_set read_fds;
fd_set write_fds;
int nfds;
for (;;)
{
FD_ZERO(&read_fds);
FD_ZERO(&write_fds);
nfds = ares_fds(channel, &read_fds,&write_fds);
if (nfds == 0)
{
break;
}
tvp = ares_timeout(channel, nullptr, &tv);
select(nfds, &read_fds, &write_fds, nullptr, tvp);
ares_process(channel, &read_fds, &write_fds);
}
}
}

View File

@ -0,0 +1,42 @@
#pragma once
#include "DNSPTRResolver.h"
using ares_channel = struct ares_channeldata *;
namespace DB
{
/*
* Implements reverse DNS resolution using c-ares lib. System reverse DNS resolution via
* gethostbyaddr or getnameinfo does not work reliably because in some systems
* it returns all PTR records for a given IP and in others it returns only one.
* */
class CaresPTRResolver : public DNSPTRResolver
{
friend class DNSPTRResolverProvider;
/*
* Allow only DNSPTRProvider to instantiate this class
* */
struct provider_token {};
public:
explicit CaresPTRResolver(provider_token);
~CaresPTRResolver() override;
std::vector<std::string> resolve(const std::string & ip) override;
std::vector<std::string> resolve_v6(const std::string & ip) override;
private:
void wait();
void resolve(const std::string & ip, std::vector<std::string> & response);
void resolve_v6(const std::string & ip, std::vector<std::string> & response);
ares_channel channel;
};
}

View File

@ -73,7 +73,7 @@ bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const S
/// Check that the right configuration has the same set of subkeys as the left configuration. /// Check that the right configuration has the same set of subkeys as the left configuration.
Poco::Util::AbstractConfiguration::Keys right_subkeys; Poco::Util::AbstractConfiguration::Keys right_subkeys;
right.keys(right_key, right_subkeys); right.keys(right_key, right_subkeys);
std::unordered_set<StringRef> left_subkeys{subkeys.begin(), subkeys.end()}; std::unordered_set<std::string_view> left_subkeys{subkeys.begin(), subkeys.end()};
if ((left_subkeys.size() != right_subkeys.size()) || (left_subkeys.size() != subkeys.size())) if ((left_subkeys.size() != right_subkeys.size()) || (left_subkeys.size() != subkeys.size()))
return false; return false;
for (const auto & right_subkey : right_subkeys) for (const auto & right_subkey : right_subkeys)

View File

@ -0,0 +1,18 @@
#pragma once
#include <string>
#include <vector>
namespace DB
{
struct DNSPTRResolver
{
virtual ~DNSPTRResolver() = default;
virtual std::vector<std::string> resolve(const std::string & ip) = 0;
virtual std::vector<std::string> resolve_v6(const std::string & ip) = 0;
};
}

View File

@ -0,0 +1,13 @@
#include "DNSPTRResolverProvider.h"
#include "CaresPTRResolver.h"
namespace DB
{
std::shared_ptr<DNSPTRResolver> DNSPTRResolverProvider::get()
{
static auto cares_resolver = std::make_shared<CaresPTRResolver>(
CaresPTRResolver::provider_token {}
);
return cares_resolver;
}
}

View File

@ -0,0 +1,18 @@
#pragma once
#include <memory>
#include "DNSPTRResolver.h"
namespace DB
{
/*
* Provides a ready-to-use DNSPTRResolver instance.
* It hides 3rd party lib dependencies, handles initialization and lifetime.
* Since `get` function is static, it can be called from any context. Including cached static functions.
* */
class DNSPTRResolverProvider
{
public:
static std::shared_ptr<DNSPTRResolver> get();
};
}

View File

@ -12,6 +12,7 @@
#include <atomic> #include <atomic>
#include <optional> #include <optional>
#include <string_view> #include <string_view>
#include "DNSPTRResolverProvider.h"
namespace ProfileEvents namespace ProfileEvents
{ {
@ -138,16 +139,17 @@ static DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host)
return addresses; return addresses;
} }
static String reverseResolveImpl(const Poco::Net::IPAddress & address) static Strings reverseResolveImpl(const Poco::Net::IPAddress & address)
{ {
Poco::Net::SocketAddress sock_addr(address, 0); auto ptr_resolver = DB::DNSPTRResolverProvider::get();
/// Resolve by hand, because Poco::Net::DNS::hostByAddress(...) does getaddrinfo(...) after getnameinfo(...) if (address.family() == Poco::Net::IPAddress::Family::IPv4)
char host[1024]; {
int err = getnameinfo(sock_addr.addr(), sock_addr.length(), host, sizeof(host), nullptr, 0, NI_NAMEREQD); return ptr_resolver->resolve(address.toString());
if (err) } else
throw Exception("Cannot getnameinfo(" + address.toString() + "): " + gai_strerror(err), ErrorCodes::DNS_ERROR); {
return host; return ptr_resolver->resolve_v6(address.toString());
}
} }
struct DNSResolver::Impl struct DNSResolver::Impl
@ -235,7 +237,7 @@ std::vector<Poco::Net::SocketAddress> DNSResolver::resolveAddressList(const std:
return addresses; return addresses;
} }
String DNSResolver::reverseResolve(const Poco::Net::IPAddress & address) Strings DNSResolver::reverseResolve(const Poco::Net::IPAddress & address)
{ {
if (impl->disable_cache) if (impl->disable_cache)
return reverseResolveImpl(address); return reverseResolveImpl(address);

View File

@ -36,8 +36,8 @@ public:
std::vector<Poco::Net::SocketAddress> resolveAddressList(const std::string & host, UInt16 port); std::vector<Poco::Net::SocketAddress> resolveAddressList(const std::string & host, UInt16 port);
/// Accepts host IP and resolves its host name /// Accepts host IP and resolves its host names
String reverseResolve(const Poco::Net::IPAddress & address); Strings reverseResolve(const Poco::Net::IPAddress & address);
/// Get this server host name /// Get this server host name
String getHostName(); String getHostName();

View File

@ -122,9 +122,6 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_)
values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor; values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor;
values.amount_of_offset_change_value = (transition.to - transition.from) / Values::OffsetChangeFactor; values.amount_of_offset_change_value = (transition.to - transition.from) / Values::OffsetChangeFactor;
// std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n";
// std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n";
/// We don't support too large changes. /// We don't support too large changes.
if (values.amount_of_offset_change_value > 24 * 4) if (values.amount_of_offset_change_value > 24 * 4)
values.amount_of_offset_change_value = 24 * 4; values.amount_of_offset_change_value = 24 * 4;

View File

@ -74,7 +74,6 @@ public:
using key_type = Key; using key_type = Key;
using mapped_type = typename Cell::mapped_type; using mapped_type = typename Cell::mapped_type;
using value_type = typename Cell::value_type; using value_type = typename Cell::value_type;
using cell_type = Cell;
class Reader final : private Cell::State class Reader final : private Cell::State
{ {
@ -247,39 +246,6 @@ public:
} }
} }
/// Same, but return false if it's full.
bool ALWAYS_INLINE tryEmplace(Key x, iterator & it, bool & inserted)
{
Cell * res = findCell(x);
it = iteratorTo(res);
inserted = res == buf + m_size;
if (inserted)
{
if (res == buf + capacity)
return false;
new(res) Cell(x, *this);
++m_size;
}
return true;
}
/// Copy the cell from another hash table. It is assumed that there was no such key in the table yet.
void ALWAYS_INLINE insertUnique(const Cell * cell)
{
memcpy(&buf[m_size], cell, sizeof(*cell));
++m_size;
}
void ALWAYS_INLINE insertUnique(Key x)
{
new(&buf[m_size]) Cell(x, *this);
++m_size;
}
iterator ALWAYS_INLINE find(Key x) { return iteratorTo(findCell(x)); } iterator ALWAYS_INLINE find(Key x) { return iteratorTo(findCell(x)); }
const_iterator ALWAYS_INLINE find(Key x) const { return iteratorTo(findCell(x)); } const_iterator ALWAYS_INLINE find(Key x) const { return iteratorTo(findCell(x)); }
@ -381,36 +347,3 @@ template
> >
using SmallSet = SmallTable<Key, HashTableCell<Key, HashUnused>, capacity>; using SmallSet = SmallTable<Key, HashTableCell<Key, HashUnused>, capacity>;
template
<
typename Key,
typename Cell,
size_t capacity
>
class SmallMapTable : public SmallTable<Key, Cell, capacity>
{
public:
using key_type = Key;
using mapped_type = typename Cell::mapped_type;
using value_type = typename Cell::value_type;
using cell_type = Cell;
mapped_type & ALWAYS_INLINE operator[](Key x)
{
typename SmallMapTable::iterator it;
bool inserted;
this->emplace(x, it, inserted);
new (&it->getMapped()) mapped_type();
return it->getMapped();
}
};
template
<
typename Key,
typename Mapped,
size_t capacity
>
using SmallMap = SmallMapTable<Key, HashMapCell<Key, Mapped, HashUnused>, capacity>;

View File

@ -45,7 +45,7 @@ void LRUFileCache::initialize()
catch (...) catch (...)
{ {
tryLogCurrentException(__PRETTY_FUNCTION__); tryLogCurrentException(__PRETTY_FUNCTION__);
return; throw;
} }
} }
else else
@ -841,7 +841,11 @@ void LRUFileCache::loadCacheInfoIntoMemory(std::lock_guard<std::mutex> & cache_l
/// cache_base_path / key_prefix / key / offset /// cache_base_path / key_prefix / key / offset
if (!files.empty()) if (!files.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache already initialized"); throw Exception(
ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
"Cache initialization is partially made. "
"This can be a result of a failed first attempt to initialize cache. "
"Please, check log for error messages");
fs::directory_iterator key_prefix_it{cache_base_path}; fs::directory_iterator key_prefix_it{cache_base_path};
for (; key_prefix_it != fs::directory_iterator(); ++key_prefix_it) for (; key_prefix_it != fs::directory_iterator(); ++key_prefix_it)

View File

@ -342,6 +342,23 @@ OptimizedRegularExpressionImpl<thread_safe>::OptimizedRegularExpressionImpl(cons
} }
} }
template <bool thread_safe>
OptimizedRegularExpressionImpl<thread_safe>::OptimizedRegularExpressionImpl(OptimizedRegularExpressionImpl && rhs) noexcept
: is_trivial(rhs.is_trivial)
, required_substring_is_prefix(rhs.required_substring_is_prefix)
, is_case_insensitive(rhs.is_case_insensitive)
, required_substring(std::move(rhs.required_substring))
, re2(std::move(rhs.re2))
, number_of_subpatterns(rhs.number_of_subpatterns)
{
if (!required_substring.empty())
{
if (is_case_insensitive)
case_insensitive_substring_searcher.emplace(required_substring.data(), required_substring.size());
else
case_sensitive_substring_searcher.emplace(required_substring.data(), required_substring.size());
}
}
template <bool thread_safe> template <bool thread_safe>
bool OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject, size_t subject_size) const bool OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject, size_t subject_size) const

View File

@ -56,6 +56,9 @@ public:
using StringPieceType = std::conditional_t<thread_safe, re2::StringPiece, re2_st::StringPiece>; using StringPieceType = std::conditional_t<thread_safe, re2::StringPiece, re2_st::StringPiece>;
OptimizedRegularExpressionImpl(const std::string & regexp_, int options = 0); /// NOLINT OptimizedRegularExpressionImpl(const std::string & regexp_, int options = 0); /// NOLINT
/// StringSearcher store pointers to required_substring, it must be updated on move.
OptimizedRegularExpressionImpl(OptimizedRegularExpressionImpl && rhs) noexcept;
OptimizedRegularExpressionImpl(const OptimizedRegularExpressionImpl & rhs) = delete;
bool match(const std::string & subject) const bool match(const std::string & subject) const
{ {

View File

@ -355,8 +355,6 @@ private:
template <size_t PASS> template <size_t PASS>
static inline void radixSortMSDInternal(Element * arr, size_t size, size_t limit) static inline void radixSortMSDInternal(Element * arr, size_t size, size_t limit)
{ {
// std::cerr << PASS << ", " << size << ", " << limit << "\n";
/// The beginning of every i-1-th bucket. 0th element will be equal to 1st. /// The beginning of every i-1-th bucket. 0th element will be equal to 1st.
/// Last element will point to array end. /// Last element will point to array end.
std::unique_ptr<Element *[]> prev_buckets{new Element*[HISTOGRAM_SIZE + 1]}; std::unique_ptr<Element *[]> prev_buckets{new Element*[HISTOGRAM_SIZE + 1]};

View File

@ -3,6 +3,7 @@
#include <memory> #include <memory>
#include <IO/ReadBufferFromFile.h> #include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromFile.h> #include <IO/WriteBufferFromFile.h>
#include <unordered_map>
namespace DB namespace DB

View File

@ -42,13 +42,14 @@ void TraceSender::send(TraceType trace_type, const StackTrace & stack_trace, Int
char buffer[buf_size]; char buffer[buf_size];
WriteBufferFromFileDescriptorDiscardOnFailure out(pipe.fds_rw[1], buf_size, buffer); WriteBufferFromFileDescriptorDiscardOnFailure out(pipe.fds_rw[1], buf_size, buffer);
StringRef query_id; std::string_view query_id;
UInt64 thread_id; UInt64 thread_id;
if (CurrentThread::isInitialized()) if (CurrentThread::isInitialized())
{ {
query_id = StringRef(CurrentThread::getQueryId()); query_id = CurrentThread::getQueryId();
query_id.size = std::min(query_id.size, QUERY_ID_MAX_LEN); if (query_id.size() > QUERY_ID_MAX_LEN)
query_id.remove_suffix(query_id.size() - QUERY_ID_MAX_LEN);
thread_id = CurrentThread::get().thread_id; thread_id = CurrentThread::get().thread_id;
} }
@ -59,8 +60,8 @@ void TraceSender::send(TraceType trace_type, const StackTrace & stack_trace, Int
writeChar(false, out); /// true if requested to stop the collecting thread. writeChar(false, out); /// true if requested to stop the collecting thread.
writeBinary(static_cast<uint8_t>(query_id.size), out); writeBinary(static_cast<uint8_t>(query_id.size()), out);
out.write(query_id.data, query_id.size); out.write(query_id.data(), query_id.size());
size_t stack_trace_size = stack_trace.getSize(); size_t stack_trace_size = stack_trace.getSize();
size_t stack_trace_offset = stack_trace.getOffset(); size_t stack_trace_offset = stack_trace.getOffset();

View File

@ -9,7 +9,6 @@
#include <Common/StringSearcher.h> #include <Common/StringSearcher.h>
#include <Common/StringUtils/StringUtils.h> #include <Common/StringUtils/StringUtils.h>
#include <Common/UTF8Helpers.h> #include <Common/UTF8Helpers.h>
#include <base/StringRef.h>
#include <base/unaligned.h> #include <base/unaligned.h>
/** Search for a substring in a string by Volnitsky's algorithm /** Search for a substring in a string by Volnitsky's algorithm

View File

@ -99,6 +99,7 @@ try
std::cout << "list\n"; std::cout << "list\n";
zk.list("/", zk.list("/",
Coordination::ListRequestType::ALL,
[&](const ListResponse & response) [&](const ListResponse & response)
{ {
if (response.error != Coordination::Error::ZOK) if (response.error != Coordination::Error::ZOK)

87
src/Common/base58.h Normal file
View File

@ -0,0 +1,87 @@
#pragma once
#include <climits>
#include <cstring>
namespace DB
{
inline size_t encodeBase58(const char8_t * src, char8_t * dst)
{
const char * base58_encoding_alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
size_t idx = 0;
for (; *src; ++src)
{
unsigned int carry = static_cast<unsigned char>(*src);
for (size_t j = 0; j < idx; ++j)
{
carry += static_cast<unsigned int>(dst[j] << 8);
dst[j] = static_cast<unsigned char>(carry % 58);
carry /= 58;
}
while (carry > 0)
{
dst[idx++] = static_cast<unsigned char>(carry % 58);
carry /= 58;
}
}
size_t c_idx = idx >> 1;
for (size_t i = 0; i < c_idx; ++i)
{
char s = base58_encoding_alphabet[static_cast<unsigned char>(dst[i])];
dst[i] = base58_encoding_alphabet[static_cast<unsigned char>(dst[idx - (i + 1)])];
dst[idx - (i + 1)] = s;
}
if ((idx & 1))
{
dst[c_idx] = base58_encoding_alphabet[static_cast<unsigned char>(dst[c_idx])];
}
dst[idx] = '\0';
return idx + 1;
}
inline size_t decodeBase58(const char8_t * src, char8_t * dst)
{
const signed char uint_max = UINT_MAX;
const signed char map_digits[128]
= {uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max,
uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max,
uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max,
uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, 0, 1, 2, 3, 4, 5, 6, 7, 8, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, 9, 10, 11, 12, 13, 14, 15, 16, uint_max, 17, 18, 19, 20, 21, uint_max, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
uint_max, uint_max, uint_max, uint_max, uint_max, uint_max, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, uint_max, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, uint_max, uint_max, uint_max, uint_max, uint_max};
size_t idx = 0;
for (; *src; ++src)
{
unsigned int carry = map_digits[*src];
if (unlikely(carry == UINT_MAX))
{
return 0;
}
for (size_t j = 0; j < idx; ++j)
{
carry += static_cast<unsigned char>(dst[j]) * 58;
dst[j] = static_cast<unsigned char>(carry & 0xff);
carry >>= 8;
}
while (carry > 0)
{
dst[idx++] = static_cast<unsigned char>(carry & 0xff);
carry >>= 8;
}
}
size_t c_idx = idx >> 1;
for (size_t i = 0; i < c_idx; ++i)
{
char s = dst[i];
dst[i] = dst[idx - (i + 1)];
dst[idx - (i + 1)] = s;
}
dst[idx] = '\0';
return idx + 1;
}
}

View File

@ -1,5 +1,4 @@
#include <iostream> #include <iostream>
#include <iomanip>
#include <Interpreters/AggregationCommon.h> #include <Interpreters/AggregationCommon.h>
@ -33,22 +32,6 @@ int main(int, char **)
std::cerr << "dump: " << wb.str() << std::endl; std::cerr << "dump: " << wb.str() << std::endl;
} }
{
using Cont = SmallMap<int, std::string, 16>;
Cont cont;
cont.insert(Cont::value_type(1, "Hello, world!"));
cont[1] = "Goodbye.";
for (auto x : cont)
std::cerr << x.getKey() << " -> " << x.getMapped() << std::endl;
DB::WriteBufferFromOwnString wb;
cont.writeText(wb);
std::cerr << "dump: " << wb.str() << std::endl;
}
{ {
using Cont = SmallSet<DB::UInt128, 16>; using Cont = SmallSet<DB::UInt128, 16>;
Cont cont; Cont cont;

View File

@ -92,6 +92,8 @@ static int pollPid(pid_t pid, int timeout_in_ms)
} }
#elif defined(OS_DARWIN) || defined(OS_FREEBSD) #elif defined(OS_DARWIN) || defined(OS_FREEBSD)
#pragma clang diagnostic ignored "-Wreserved-identifier"
#include <sys/event.h> #include <sys/event.h>
#include <err.h> #include <err.h>

View File

@ -106,21 +106,15 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c
throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH);
} }
static void readHeaderAndGetCodecAndSize(
/// Read compressed data into compressed_buffer. Get size of decompressed data from block header. Checksum if need. const char * compressed_buffer,
/// Returns number of compressed bytes read. UInt8 header_size,
size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed, size_t & size_compressed_without_checksum, bool always_copy) CompressionCodecPtr & codec,
size_t & size_decompressed,
size_t & size_compressed_without_checksum,
bool allow_different_codecs)
{ {
if (compressed_in->eof()) uint8_t method = ICompressionCodec::readMethod(compressed_buffer);
return 0;
UInt8 header_size = ICompressionCodec::getHeaderSize();
own_compressed_buffer.resize(header_size + sizeof(Checksum));
compressed_in->readStrict(own_compressed_buffer.data(), sizeof(Checksum) + header_size);
char * compressed_header = own_compressed_buffer.data() + sizeof(Checksum);
uint8_t method = ICompressionCodec::readMethod(compressed_header);
if (!codec) if (!codec)
{ {
@ -142,8 +136,8 @@ size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed,
} }
} }
size_compressed_without_checksum = ICompressionCodec::readCompressedBlockSize(compressed_header); size_compressed_without_checksum = ICompressionCodec::readCompressedBlockSize(compressed_buffer);
size_decompressed = ICompressionCodec::readDecompressedBlockSize(compressed_header); size_decompressed = ICompressionCodec::readDecompressedBlockSize(compressed_buffer);
/// This is for clang static analyzer. /// This is for clang static analyzer.
assert(size_decompressed > 0); assert(size_decompressed > 0);
@ -157,8 +151,27 @@ size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed,
if (size_compressed_without_checksum < header_size) if (size_compressed_without_checksum < header_size)
throw Exception("Can't decompress data: the compressed data size (" + toString(size_compressed_without_checksum) throw Exception("Can't decompress data: the compressed data size (" + toString(size_compressed_without_checksum)
+ ", this should include header size) is less than the header size (" + toString(header_size) + ")", ErrorCodes::CORRUPTED_DATA); + ", this should include header size) is less than the header size (" + toString(header_size) + ")", ErrorCodes::CORRUPTED_DATA);
}
ProfileEvents::increment(ProfileEvents::ReadCompressedBytes, size_compressed_without_checksum + sizeof(Checksum)); /// Read compressed data into compressed_buffer. Get size of decompressed data from block header. Checksum if need.
/// Returns number of compressed bytes read.
size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed, size_t & size_compressed_without_checksum, bool always_copy)
{
if (compressed_in->eof())
return 0;
UInt8 header_size = ICompressionCodec::getHeaderSize();
own_compressed_buffer.resize(header_size + sizeof(Checksum));
compressed_in->readStrict(own_compressed_buffer.data(), sizeof(Checksum) + header_size);
readHeaderAndGetCodecAndSize(
own_compressed_buffer.data() + sizeof(Checksum),
header_size,
codec,
size_decompressed,
size_compressed_without_checksum,
allow_different_codecs);
auto additional_size_at_the_end_of_buffer = codec->getAdditionalSizeAtTheEndOfBuffer(); auto additional_size_at_the_end_of_buffer = codec->getAdditionalSizeAtTheEndOfBuffer();
@ -184,9 +197,55 @@ size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed,
validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum); validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum);
} }
ProfileEvents::increment(ProfileEvents::ReadCompressedBytes, size_compressed_without_checksum + sizeof(Checksum));
return size_compressed_without_checksum + sizeof(Checksum); return size_compressed_without_checksum + sizeof(Checksum);
} }
/// Read compressed data into compressed_buffer for asynchronous decompression to avoid the situation of "read compressed block across the compressed_in".
size_t CompressedReadBufferBase::readCompressedDataBlockForAsynchronous(size_t & size_decompressed, size_t & size_compressed_without_checksum)
{
UInt8 header_size = ICompressionCodec::getHeaderSize();
/// Make sure the whole header located in 'compressed_in->' buffer.
if (compressed_in->eof() || (compressed_in->available() < (header_size + sizeof(Checksum))))
return 0;
own_compressed_buffer.resize(header_size + sizeof(Checksum));
compressed_in->readStrict(own_compressed_buffer.data(), sizeof(Checksum) + header_size);
readHeaderAndGetCodecAndSize(
own_compressed_buffer.data() + sizeof(Checksum),
header_size,
codec,
size_decompressed,
size_compressed_without_checksum,
allow_different_codecs);
auto additional_size_at_the_end_of_buffer = codec->getAdditionalSizeAtTheEndOfBuffer();
/// Make sure the whole compressed block located in 'compressed_in->' buffer.
/// Otherwise, abandon header and restore original offset of compressed_in
if (compressed_in->offset() >= header_size + sizeof(Checksum) &&
compressed_in->available() >= (size_compressed_without_checksum - header_size) + additional_size_at_the_end_of_buffer + sizeof(Checksum))
{
compressed_in->position() -= header_size;
compressed_buffer = compressed_in->position();
compressed_in->position() += size_compressed_without_checksum;
if (!disable_checksum)
{
Checksum & checksum = *reinterpret_cast<Checksum *>(own_compressed_buffer.data());
validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum);
}
ProfileEvents::increment(ProfileEvents::ReadCompressedBytes, size_compressed_without_checksum + sizeof(Checksum));
return size_compressed_without_checksum + sizeof(Checksum);
}
else
{
compressed_in->position() -= (sizeof(Checksum) + header_size);
return 0;
}
}
static void readHeaderAndGetCodec(const char * compressed_buffer, size_t size_decompressed, CompressionCodecPtr & codec, bool allow_different_codecs) static void readHeaderAndGetCodec(const char * compressed_buffer, size_t size_decompressed, CompressionCodecPtr & codec, bool allow_different_codecs)
{ {
@ -216,14 +275,12 @@ static void readHeaderAndGetCodec(const char * compressed_buffer, size_t size_de
} }
} }
void CompressedReadBufferBase::decompressTo(char * to, size_t size_decompressed, size_t size_compressed_without_checksum) void CompressedReadBufferBase::decompressTo(char * to, size_t size_decompressed, size_t size_compressed_without_checksum)
{ {
readHeaderAndGetCodec(compressed_buffer, size_decompressed, codec, allow_different_codecs); readHeaderAndGetCodec(compressed_buffer, size_decompressed, codec, allow_different_codecs);
codec->decompress(compressed_buffer, size_compressed_without_checksum, to); codec->decompress(compressed_buffer, size_compressed_without_checksum, to);
} }
void CompressedReadBufferBase::decompress(BufferBase::Buffer & to, size_t size_decompressed, size_t size_compressed_without_checksum) void CompressedReadBufferBase::decompress(BufferBase::Buffer & to, size_t size_decompressed, size_t size_compressed_without_checksum)
{ {
readHeaderAndGetCodec(compressed_buffer, size_decompressed, codec, allow_different_codecs); readHeaderAndGetCodec(compressed_buffer, size_decompressed, codec, allow_different_codecs);
@ -245,6 +302,17 @@ void CompressedReadBufferBase::decompress(BufferBase::Buffer & to, size_t size_d
codec->decompress(compressed_buffer, size_compressed_without_checksum, to.begin()); codec->decompress(compressed_buffer, size_compressed_without_checksum, to.begin());
} }
void CompressedReadBufferBase::flushAsynchronousDecompressRequests() const
{
if (codec)
codec->flushAsynchronousDecompressRequests();
}
void CompressedReadBufferBase::setDecompressMode(ICompressionCodec::CodecMode mode) const
{
if (codec)
codec->setDecompressMode(mode);
}
/// 'compressed_in' could be initialized lazily, but before first call of 'readCompressedData'. /// 'compressed_in' could be initialized lazily, but before first call of 'readCompressedData'.
CompressedReadBufferBase::CompressedReadBufferBase(ReadBuffer * in, bool allow_different_codecs_) CompressedReadBufferBase::CompressedReadBufferBase(ReadBuffer * in, bool allow_different_codecs_)
@ -253,7 +321,7 @@ CompressedReadBufferBase::CompressedReadBufferBase(ReadBuffer * in, bool allow_d
} }
CompressedReadBufferBase::~CompressedReadBufferBase() = default; /// Proper destruction of unique_ptr of forward-declared type. CompressedReadBufferBase::~CompressedReadBufferBase() = default; /// Proper destruction of unique_ptr of forward-declared type.
} }

View File

@ -39,6 +39,17 @@ protected:
/// Returns number of compressed bytes read. /// Returns number of compressed bytes read.
size_t readCompressedData(size_t & size_decompressed, size_t & size_compressed_without_checksum, bool always_copy); size_t readCompressedData(size_t & size_decompressed, size_t & size_compressed_without_checksum, bool always_copy);
/// Read compressed data into compressed_buffer for asynchronous decompression to avoid the situation of "read compressed block across the compressed_in".
///
/// Compressed block may not be completely contained in "compressed_in" buffer which means compressed block may be read across the "compressed_in".
/// For native LZ4/ZSTD, it has no problem in facing situation above because they are synchronous.
/// But for asynchronous decompression, such as QPL deflate, it requires source and target buffer for decompression can not be overwritten until execution complete.
///
/// Returns number of compressed bytes read.
/// If Returns value > 0, means the address range for current block are maintained in "compressed_in", then asynchronous decompression can be called to boost performance.
/// If Returns value == 0, it means current block cannot be decompressed asynchronously.Meanwhile, asynchronous requests for previous blocks should be flushed if any.
size_t readCompressedDataBlockForAsynchronous(size_t & size_decompressed, size_t & size_compressed_without_checksum);
/// Decompress into memory pointed by `to` /// Decompress into memory pointed by `to`
void decompressTo(char * to, size_t size_decompressed, size_t size_compressed_without_checksum); void decompressTo(char * to, size_t size_decompressed, size_t size_compressed_without_checksum);
@ -46,6 +57,14 @@ protected:
/// It is more efficient for compression codec NONE but not suitable if you want to decompress into specific location. /// It is more efficient for compression codec NONE but not suitable if you want to decompress into specific location.
void decompress(BufferBase::Buffer & to, size_t size_decompressed, size_t size_compressed_without_checksum); void decompress(BufferBase::Buffer & to, size_t size_decompressed, size_t size_compressed_without_checksum);
/// Flush all asynchronous decompress request.
void flushAsynchronousDecompressRequests() const;
/// Set decompression mode: Synchronous/Asynchronous/SoftwareFallback.
/// The mode is "Synchronous" by default.
/// flushAsynchronousDecompressRequests must be called subsequently once set "Asynchronous" mode.
void setDecompressMode(ICompressionCodec::CodecMode mode) const;
public: public:
/// 'compressed_in' could be initialized lazily, but before first call of 'readCompressedData'. /// 'compressed_in' could be initialized lazily, but before first call of 'readCompressedData'.
explicit CompressedReadBufferBase(ReadBuffer * in = nullptr, bool allow_different_codecs_ = false); explicit CompressedReadBufferBase(ReadBuffer * in = nullptr, bool allow_different_codecs_ = false);

View File

@ -91,6 +91,9 @@ void CompressedReadBufferFromFile::seek(size_t offset_in_compressed_file, size_t
size_t CompressedReadBufferFromFile::readBig(char * to, size_t n) size_t CompressedReadBufferFromFile::readBig(char * to, size_t n)
{ {
size_t bytes_read = 0; size_t bytes_read = 0;
/// The codec mode is only relevant for codecs which support hardware offloading.
ICompressionCodec::CodecMode decompress_mode = ICompressionCodec::CodecMode::Synchronous;
bool read_tail = false;
/// If there are unread bytes in the buffer, then we copy needed to `to`. /// If there are unread bytes in the buffer, then we copy needed to `to`.
if (pos < working_buffer.end()) if (pos < working_buffer.end())
@ -102,10 +105,28 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n)
size_t size_decompressed = 0; size_t size_decompressed = 0;
size_t size_compressed_without_checksum = 0; size_t size_compressed_without_checksum = 0;
size_t new_size_compressed = readCompressedData(size_decompressed, size_compressed_without_checksum, false); ///Try to read block which is entirely located in a single 'compressed_in->' buffer.
size_t new_size_compressed = readCompressedDataBlockForAsynchronous(size_decompressed, size_compressed_without_checksum);
if (new_size_compressed)
{
/// Current block is entirely located in a single 'compressed_in->' buffer.
/// We can set asynchronous decompression mode if supported to boost performance.
decompress_mode = ICompressionCodec::CodecMode::Asynchronous;
}
else
{
/// Current block cannot be decompressed asynchronously, means it probably span across two compressed_in buffers.
/// Meanwhile, asynchronous requests for previous blocks should be flushed if any.
flushAsynchronousDecompressRequests();
/// Fallback to generic API
new_size_compressed = readCompressedData(size_decompressed, size_compressed_without_checksum, false);
decompress_mode = ICompressionCodec::CodecMode::Synchronous;
}
size_compressed = 0; /// file_in no longer points to the end of the block in working_buffer. size_compressed = 0; /// file_in no longer points to the end of the block in working_buffer.
if (!new_size_compressed) if (!new_size_compressed)
return bytes_read; break;
auto additional_size_at_the_end_of_buffer = codec->getAdditionalSizeAtTheEndOfBuffer(); auto additional_size_at_the_end_of_buffer = codec->getAdditionalSizeAtTheEndOfBuffer();
@ -113,6 +134,7 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n)
/// need to skip some bytes in decompressed data (seek happened before readBig call). /// need to skip some bytes in decompressed data (seek happened before readBig call).
if (nextimpl_working_buffer_offset == 0 && size_decompressed + additional_size_at_the_end_of_buffer <= n - bytes_read) if (nextimpl_working_buffer_offset == 0 && size_decompressed + additional_size_at_the_end_of_buffer <= n - bytes_read)
{ {
setDecompressMode(decompress_mode);
decompressTo(to + bytes_read, size_decompressed, size_compressed_without_checksum); decompressTo(to + bytes_read, size_decompressed, size_compressed_without_checksum);
bytes_read += size_decompressed; bytes_read += size_decompressed;
bytes += size_decompressed; bytes += size_decompressed;
@ -127,6 +149,8 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n)
assert(size_decompressed + additional_size_at_the_end_of_buffer > 0); assert(size_decompressed + additional_size_at_the_end_of_buffer > 0);
memory.resize(size_decompressed + additional_size_at_the_end_of_buffer); memory.resize(size_decompressed + additional_size_at_the_end_of_buffer);
working_buffer = Buffer(memory.data(), &memory[size_decompressed]); working_buffer = Buffer(memory.data(), &memory[size_decompressed]);
/// Synchronous mode must be set since we need read partial data immediately from working buffer to target buffer.
setDecompressMode(ICompressionCodec::CodecMode::Synchronous);
decompress(working_buffer, size_decompressed, size_compressed_without_checksum); decompress(working_buffer, size_decompressed, size_compressed_without_checksum);
/// Read partial data from first block. Won't run here at second block. /// Read partial data from first block. Won't run here at second block.
@ -145,15 +169,25 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n)
assert(size_decompressed + additional_size_at_the_end_of_buffer > 0); assert(size_decompressed + additional_size_at_the_end_of_buffer > 0);
memory.resize(size_decompressed + additional_size_at_the_end_of_buffer); memory.resize(size_decompressed + additional_size_at_the_end_of_buffer);
working_buffer = Buffer(memory.data(), &memory[size_decompressed]); working_buffer = Buffer(memory.data(), &memory[size_decompressed]);
// Asynchronous mode can be set here because working_buffer wouldn't be overwritten any more since this is the last block.
setDecompressMode(ICompressionCodec::CodecMode::Asynchronous);
decompress(working_buffer, size_decompressed, size_compressed_without_checksum); decompress(working_buffer, size_decompressed, size_compressed_without_checksum);
read_tail = true;
///Read partial data from last block.
pos = working_buffer.begin();
bytes_read += read(to + bytes_read, n - bytes_read);
break; break;
} }
} }
/// Here we must make sure all asynchronous requests above are completely done.
flushAsynchronousDecompressRequests();
if (read_tail)
{
/// Manually take nextimpl_working_buffer_offset into account, because we don't use
/// nextImpl in this method.
pos = working_buffer.begin();
bytes_read += read(to + bytes_read, n - bytes_read);
}
return bytes_read; return bytes_read;
} }

View File

@ -0,0 +1,413 @@
#ifdef ENABLE_QPL_COMPRESSION
#include <cstdio>
#include <thread>
#include <Compression/CompressionCodecDeflateQpl.h>
#include <Compression/CompressionFactory.h>
#include <Compression/CompressionInfo.h>
#include <Parsers/ASTIdentifier.h>
#include <Poco/Logger.h>
#include <Common/logger_useful.h>
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_COMPRESS;
extern const int CANNOT_DECOMPRESS;
}
std::array<qpl_job *, DeflateQplJobHWPool::MAX_HW_JOB_NUMBER> DeflateQplJobHWPool::hw_job_ptr_pool;
std::array<std::atomic_bool, DeflateQplJobHWPool::MAX_HW_JOB_NUMBER> DeflateQplJobHWPool::hw_job_ptr_locks;
bool DeflateQplJobHWPool::job_pool_ready = false;
std::unique_ptr<uint8_t[]> DeflateQplJobHWPool::hw_jobs_buffer;
DeflateQplJobHWPool & DeflateQplJobHWPool::instance()
{
static DeflateQplJobHWPool pool;
return pool;
}
DeflateQplJobHWPool::DeflateQplJobHWPool()
:random_engine(std::random_device()())
,distribution(0, MAX_HW_JOB_NUMBER-1)
{
Poco::Logger * log = &Poco::Logger::get("DeflateQplJobHWPool");
UInt32 job_size = 0;
const char * qpl_version = qpl_get_library_version();
/// Get size required for saving a single qpl job object
qpl_get_job_size(qpl_path_hardware, &job_size);
/// Allocate entire buffer for storing all job objects
hw_jobs_buffer = std::make_unique<uint8_t[]>(job_size * MAX_HW_JOB_NUMBER);
/// Initialize pool for storing all job object pointers
/// Reallocate buffer by shifting address offset for each job object.
for (UInt32 index = 0; index < MAX_HW_JOB_NUMBER; ++index)
{
qpl_job * qpl_job_ptr = reinterpret_cast<qpl_job *>(hw_jobs_buffer.get() + index * job_size);
if (qpl_init_job(qpl_path_hardware, qpl_job_ptr) != QPL_STS_OK)
{
job_pool_ready = false;
LOG_WARNING(log, "Initialization of hardware-assisted DeflateQpl codec failed, falling back to software DeflateQpl codec. Please check if Intel In-Memory Analytics Accelerator (IAA) is properly set up. QPL Version: {}.",qpl_version);
return;
}
hw_job_ptr_pool[index] = qpl_job_ptr;
unLockJob(index);
}
job_pool_ready = true;
LOG_DEBUG(log, "Hardware-assisted DeflateQpl codec is ready! QPL Version: {}",qpl_version);
}
DeflateQplJobHWPool::~DeflateQplJobHWPool()
{
for (UInt32 i = 0; i < MAX_HW_JOB_NUMBER; ++i)
{
if (hw_job_ptr_pool[i])
{
while (!tryLockJob(i));
qpl_fini_job(hw_job_ptr_pool[i]);
unLockJob(i);
hw_job_ptr_pool[i] = nullptr;
}
}
job_pool_ready = false;
}
qpl_job * DeflateQplJobHWPool::acquireJob(UInt32 &job_id)
{
if (isJobPoolReady())
{
UInt32 retry = 0;
auto index = distribution(random_engine);
while (!tryLockJob(index))
{
index = distribution(random_engine);
retry++;
if (retry > MAX_HW_JOB_NUMBER)
{
return nullptr;
}
}
job_id = MAX_HW_JOB_NUMBER - index;
assert(index < MAX_HW_JOB_NUMBER);
return hw_job_ptr_pool[index];
}
else
return nullptr;
}
void DeflateQplJobHWPool::releaseJob(UInt32 job_id)
{
if (isJobPoolReady())
unLockJob(MAX_HW_JOB_NUMBER - job_id);
}
bool DeflateQplJobHWPool::tryLockJob(UInt32 index)
{
bool expected = false;
assert(index < MAX_HW_JOB_NUMBER);
return hw_job_ptr_locks[index].compare_exchange_strong(expected, true);
}
void DeflateQplJobHWPool::unLockJob(UInt32 index)
{
assert(index < MAX_HW_JOB_NUMBER);
hw_job_ptr_locks[index].store(false);
}
//HardwareCodecDeflateQpl
HardwareCodecDeflateQpl::HardwareCodecDeflateQpl()
:log(&Poco::Logger::get("HardwareCodecDeflateQpl"))
{
}
HardwareCodecDeflateQpl::~HardwareCodecDeflateQpl()
{
#ifndef NDEBUG
assert(decomp_async_job_map.empty());
#else
if (!decomp_async_job_map.empty())
{
LOG_WARNING(log, "Find un-released job when HardwareCodecDeflateQpl destroy");
for (auto it : decomp_async_job_map)
{
DeflateQplJobHWPool::instance().releaseJob(it.first);
}
decomp_async_job_map.clear();
}
#endif
}
Int32 HardwareCodecDeflateQpl::doCompressData(const char * source, UInt32 source_size, char * dest, UInt32 dest_size) const
{
UInt32 job_id = 0;
qpl_job* job_ptr = nullptr;
UInt32 compressed_size = 0;
if (!(job_ptr = DeflateQplJobHWPool::instance().acquireJob(job_id)))
{
LOG_INFO(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doCompressData->acquireJob fail, probably job pool exhausted)");
return RET_ERROR;
}
job_ptr->op = qpl_op_compress;
job_ptr->next_in_ptr = reinterpret_cast<uint8_t *>(const_cast<char *>(source));
job_ptr->next_out_ptr = reinterpret_cast<uint8_t *>(dest);
job_ptr->available_in = source_size;
job_ptr->level = qpl_default_level;
job_ptr->available_out = dest_size;
job_ptr->flags = QPL_FLAG_FIRST | QPL_FLAG_DYNAMIC_HUFFMAN | QPL_FLAG_LAST | QPL_FLAG_OMIT_VERIFY;
if (auto status = qpl_execute_job(job_ptr); status == QPL_STS_OK)
{
compressed_size = job_ptr->total_out;
DeflateQplJobHWPool::instance().releaseJob(job_id);
return compressed_size;
}
else
{
LOG_WARNING(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doCompressData->qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
DeflateQplJobHWPool::instance().releaseJob(job_id);
return RET_ERROR;
}
}
Int32 HardwareCodecDeflateQpl::doDecompressDataSynchronous(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size)
{
UInt32 job_id = 0;
qpl_job * job_ptr = nullptr;
UInt32 decompressed_size = 0;
if (!(job_ptr = DeflateQplJobHWPool::instance().acquireJob(job_id)))
{
LOG_INFO(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doDecompressDataSynchronous->acquireJob fail, probably job pool exhausted)");
return RET_ERROR;
}
// Performing a decompression operation
job_ptr->op = qpl_op_decompress;
job_ptr->next_in_ptr = reinterpret_cast<uint8_t *>(const_cast<char *>(source));
job_ptr->next_out_ptr = reinterpret_cast<uint8_t *>(dest);
job_ptr->available_in = source_size;
job_ptr->available_out = uncompressed_size;
job_ptr->flags = QPL_FLAG_FIRST | QPL_FLAG_LAST;
if (auto status = qpl_submit_job(job_ptr); status != QPL_STS_OK)
{
DeflateQplJobHWPool::instance().releaseJob(job_id);
LOG_WARNING(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doDecompressDataSynchronous->qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
return RET_ERROR;
}
/// Busy waiting till job complete.
do
{
_tpause(1, __rdtsc() + 1000);
} while (qpl_check_job(job_ptr) == QPL_STS_BEING_PROCESSED);
decompressed_size = job_ptr->total_out;
DeflateQplJobHWPool::instance().releaseJob(job_id);
return decompressed_size;
}
Int32 HardwareCodecDeflateQpl::doDecompressDataAsynchronous(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size)
{
UInt32 job_id = 0;
qpl_job * job_ptr = nullptr;
if (!(job_ptr = DeflateQplJobHWPool::instance().acquireJob(job_id)))
{
LOG_INFO(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doDecompressDataAsynchronous->acquireJob fail, probably job pool exhausted)");
return RET_ERROR;
}
// Performing a decompression operation
job_ptr->op = qpl_op_decompress;
job_ptr->next_in_ptr = reinterpret_cast<uint8_t *>(const_cast<char *>(source));
job_ptr->next_out_ptr = reinterpret_cast<uint8_t *>(dest);
job_ptr->available_in = source_size;
job_ptr->available_out = uncompressed_size;
job_ptr->flags = QPL_FLAG_FIRST | QPL_FLAG_LAST;
if (auto status = qpl_submit_job(job_ptr); status == QPL_STS_OK)
{
decomp_async_job_map.insert({job_id, job_ptr});
return job_id;
}
else
{
DeflateQplJobHWPool::instance().releaseJob(job_id);
LOG_WARNING(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doDecompressDataAsynchronous->qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
return RET_ERROR;
}
}
void HardwareCodecDeflateQpl::flushAsynchronousDecompressRequests()
{
UInt32 n_jobs_processing = decomp_async_job_map.size();
std::map<UInt32, qpl_job *>::iterator it = decomp_async_job_map.begin();
while (n_jobs_processing)
{
UInt32 job_id = 0;
qpl_job * job_ptr = nullptr;
job_id = it->first;
job_ptr = it->second;
if (qpl_check_job(job_ptr) == QPL_STS_BEING_PROCESSED)
{
it++;
}
else
{
it = decomp_async_job_map.erase(it);
DeflateQplJobHWPool::instance().releaseJob(job_id);
n_jobs_processing--;
if (n_jobs_processing <= 0)
break;
}
if (it == decomp_async_job_map.end())
{
it = decomp_async_job_map.begin();
_tpause(1, __rdtsc() + 1000);
}
}
}
SoftwareCodecDeflateQpl::~SoftwareCodecDeflateQpl()
{
if (!sw_job)
qpl_fini_job(sw_job);
}
qpl_job * SoftwareCodecDeflateQpl::getJobCodecPtr()
{
if (!sw_job)
{
UInt32 size = 0;
qpl_get_job_size(qpl_path_software, &size);
sw_buffer = std::make_unique<uint8_t[]>(size);
sw_job = reinterpret_cast<qpl_job *>(sw_buffer.get());
// Job initialization
if (auto status = qpl_init_job(qpl_path_software, sw_job); status != QPL_STS_OK)
throw Exception(ErrorCodes::CANNOT_COMPRESS,
"Initialization of DeflateQpl software fallback codec failed. (Details: qpl_init_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
}
return sw_job;
}
UInt32 SoftwareCodecDeflateQpl::doCompressData(const char * source, UInt32 source_size, char * dest, UInt32 dest_size)
{
qpl_job * job_ptr = getJobCodecPtr();
// Performing a compression operation
job_ptr->op = qpl_op_compress;
job_ptr->next_in_ptr = reinterpret_cast<uint8_t *>(const_cast<char *>(source));
job_ptr->next_out_ptr = reinterpret_cast<uint8_t *>(dest);
job_ptr->available_in = source_size;
job_ptr->available_out = dest_size;
job_ptr->level = qpl_default_level;
job_ptr->flags = QPL_FLAG_FIRST | QPL_FLAG_DYNAMIC_HUFFMAN | QPL_FLAG_LAST | QPL_FLAG_OMIT_VERIFY;
if (auto status = qpl_execute_job(job_ptr); status != QPL_STS_OK)
throw Exception(ErrorCodes::CANNOT_COMPRESS,
"Execution of DeflateQpl software fallback codec failed. (Details: qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
return job_ptr->total_out;
}
void SoftwareCodecDeflateQpl::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size)
{
qpl_job * job_ptr = getJobCodecPtr();
// Performing a decompression operation
job_ptr->op = qpl_op_decompress;
job_ptr->next_in_ptr = reinterpret_cast<uint8_t *>(const_cast<char *>(source));
job_ptr->next_out_ptr = reinterpret_cast<uint8_t *>(dest);
job_ptr->available_in = source_size;
job_ptr->available_out = uncompressed_size;
job_ptr->flags = QPL_FLAG_FIRST | QPL_FLAG_LAST;
if (auto status = qpl_execute_job(job_ptr); status != QPL_STS_OK)
throw Exception(ErrorCodes::CANNOT_DECOMPRESS,
"Execution of DeflateQpl software fallback codec failed. (Details: qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
}
//CompressionCodecDeflateQpl
CompressionCodecDeflateQpl::CompressionCodecDeflateQpl()
:hw_codec(std::make_unique<HardwareCodecDeflateQpl>())
,sw_codec(std::make_unique<SoftwareCodecDeflateQpl>())
{
setCodecDescription("DEFLATE_QPL");
}
uint8_t CompressionCodecDeflateQpl::getMethodByte() const
{
return static_cast<uint8_t>(CompressionMethodByte::DeflateQpl);
}
void CompressionCodecDeflateQpl::updateHash(SipHash & hash) const
{
getCodecDesc()->updateTreeHash(hash);
}
UInt32 CompressionCodecDeflateQpl::getMaxCompressedDataSize(UInt32 uncompressed_size) const
{
/// Aligned with ZLIB
return ((uncompressed_size) + ((uncompressed_size) >> 12) + ((uncompressed_size) >> 14) + ((uncompressed_size) >> 25) + 13);
}
UInt32 CompressionCodecDeflateQpl::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
Int32 res = HardwareCodecDeflateQpl::RET_ERROR;
if (DeflateQplJobHWPool::instance().isJobPoolReady())
res = hw_codec->doCompressData(source, source_size, dest, getMaxCompressedDataSize(source_size));
if (res == HardwareCodecDeflateQpl::RET_ERROR)
res = sw_codec->doCompressData(source, source_size, dest, getMaxCompressedDataSize(source_size));
return res;
}
void CompressionCodecDeflateQpl::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
{
switch (getDecompressMode())
{
case CodecMode::Synchronous:
{
Int32 res = HardwareCodecDeflateQpl::RET_ERROR;
if (DeflateQplJobHWPool::instance().isJobPoolReady())
{
res = hw_codec->doDecompressDataSynchronous(source, source_size, dest, uncompressed_size);
if (res == HardwareCodecDeflateQpl::RET_ERROR)
sw_codec->doDecompressData(source, source_size, dest, uncompressed_size);
}
else
sw_codec->doDecompressData(source, source_size, dest, uncompressed_size);
return;
}
case CodecMode::Asynchronous:
{
Int32 res = HardwareCodecDeflateQpl::RET_ERROR;
if (DeflateQplJobHWPool::instance().isJobPoolReady())
res = hw_codec->doDecompressDataAsynchronous(source, source_size, dest, uncompressed_size);
if (res == HardwareCodecDeflateQpl::RET_ERROR)
sw_codec->doDecompressData(source, source_size, dest, uncompressed_size);
return;
}
case CodecMode::SoftwareFallback:
sw_codec->doDecompressData(source, source_size, dest, uncompressed_size);
return;
}
__builtin_unreachable();
}
void CompressionCodecDeflateQpl::flushAsynchronousDecompressRequests()
{
if (DeflateQplJobHWPool::instance().isJobPoolReady())
hw_codec->flushAsynchronousDecompressRequests();
/// After flush previous all async requests, we must restore mode to be synchronous by default.
setDecompressMode(CodecMode::Synchronous);
}
void registerCodecDeflateQpl(CompressionCodecFactory & factory)
{
factory.registerSimpleCompressionCodec(
"DEFLATE_QPL", static_cast<char>(CompressionMethodByte::DeflateQpl), [&]() { return std::make_shared<CompressionCodecDeflateQpl>(); });
}
}
#endif

View File

@ -0,0 +1,120 @@
#pragma once
#include <Compression/ICompressionCodec.h>
#include <qpl/qpl.h>
#include <random>
namespace Poco
{
class Logger;
}
namespace DB
{
/// DeflateQplJobHWPool is resource pool to provide the job objects.
/// Job object is used for storing context information during offloading compression job to HW Accelerator.
class DeflateQplJobHWPool
{
public:
DeflateQplJobHWPool();
~DeflateQplJobHWPool();
qpl_job * acquireJob(UInt32 &job_id);
static void releaseJob(UInt32 job_id);
static const bool & isJobPoolReady() { return job_pool_ready; }
static DeflateQplJobHWPool & instance();
private:
static bool tryLockJob(UInt32 index);
static void unLockJob(UInt32 index);
/// Maximum jobs running in parallel supported by IAA hardware
static constexpr auto MAX_HW_JOB_NUMBER = 1024;
/// Entire buffer for storing all job objects
static std::unique_ptr<uint8_t[]> hw_jobs_buffer;
/// Job pool for storing all job object pointers
static std::array<qpl_job *, DeflateQplJobHWPool::MAX_HW_JOB_NUMBER> hw_job_ptr_pool;
/// Locks for accessing each job object pointers
static std::array<std::atomic_bool, DeflateQplJobHWPool::MAX_HW_JOB_NUMBER> hw_job_ptr_locks;
static bool job_pool_ready;
std::mt19937 random_engine;
std::uniform_int_distribution<int> distribution;
};
class SoftwareCodecDeflateQpl
{
public:
~SoftwareCodecDeflateQpl();
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest, UInt32 dest_size);
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size);
private:
qpl_job * sw_job = nullptr;
std::unique_ptr<uint8_t[]> sw_buffer;
qpl_job * getJobCodecPtr();
};
class HardwareCodecDeflateQpl
{
public:
/// RET_ERROR stands for hardware codec fail,need fallback to software codec.
static constexpr Int32 RET_ERROR = -1;
HardwareCodecDeflateQpl();
~HardwareCodecDeflateQpl();
Int32 doCompressData(const char * source, UInt32 source_size, char * dest, UInt32 dest_size) const;
///Submit job request to the IAA hardware and then busy waiting till it complete.
Int32 doDecompressDataSynchronous(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size);
///Submit job request to the IAA hardware and return immediately. IAA hardware will process decompression jobs automatically.
Int32 doDecompressDataAsynchronous(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size);
/// Flush result for all previous requests which means busy waiting till all the jobs in "decomp_async_job_map" are finished.
/// Must be called subsequently after several calls of doDecompressDataReq.
void flushAsynchronousDecompressRequests();
private:
/// Asynchronous job map for decompression: job ID - job object.
/// For each submission, push job ID && job object into this map;
/// For flush, pop out job ID && job object from this map. Use job ID to release job lock and use job object to check job status till complete.
std::map<UInt32, qpl_job *> decomp_async_job_map;
Poco::Logger * log;
};
class CompressionCodecDeflateQpl : public ICompressionCodec
{
public:
CompressionCodecDeflateQpl();
uint8_t getMethodByte() const override;
void updateHash(SipHash & hash) const override;
protected:
bool isCompression() const override
{
return true;
}
bool isGenericCompression() const override
{
return true;
}
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
///Flush result for previous asynchronous decompression requests on asynchronous mode.
void flushAsynchronousDecompressRequests() override;
private:
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
std::unique_ptr<HardwareCodecDeflateQpl> hw_codec;
std::unique_ptr<SoftwareCodecDeflateQpl> sw_codec;
};
}

Some files were not shown because too many files have changed in this diff Show More