thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix-race-storage-s3

Browse Source
This commit is contained in:
Alexey Milovidov 2023-04-03 06:25:44 +03:00 committed by GitHub
commit 99ed3b627b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
144 changed files with 4653 additions and 806 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

787
.github/workflows/master.yml vendored
View File

@ -1131,7 +1131,7 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=2
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1167,6 +1167,114 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestReleaseDatabaseReplicated2:
needs: [BuilderDebRelease]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_database_replicated
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, DatabaseReplicated)
REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestReleaseDatabaseReplicated3:
needs: [BuilderDebRelease]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_database_replicated
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, DatabaseReplicated)
REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestReleaseS3_0:
needs: [BuilderDebRelease]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_s3_storage
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (release, s3 storage)
REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=2
EOF
- name: Download json reports
@ -1190,7 +1298,7 @@ jobs:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestReleaseS3:
FunctionalStatelessTestReleaseS3_1:
needs: [BuilderDebRelease]
runs-on: [self-hosted, func-tester]
steps:
@ -1202,6 +1310,8 @@ jobs:
CHECK_NAME=Stateless tests (release, s3 storage)
REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=2
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1271,7 +1381,7 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=2
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1307,7 +1417,79 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=2
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestAsan2:
needs: [BuilderDebAsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (asan)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestAsan3:
needs: [BuilderDebAsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (asan)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1343,7 +1525,7 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=5
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1379,7 +1561,7 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=5
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1415,7 +1597,7 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=5
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1438,7 +1620,79 @@ jobs:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestUBsan:
FunctionalStatelessTestTsan3:
needs: [BuilderDebTsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (tsan)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=5
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestTsan4:
needs: [BuilderDebTsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (tsan)
REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=4
RUN_BY_HASH_TOTAL=5
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestUBsan0:
needs: [BuilderDebUBsan]
runs-on: [self-hosted, func-tester]
steps:
@ -1450,6 +1704,44 @@ jobs:
CHECK_NAME=Stateless tests (ubsan)
REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=2
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestUBsan1:
needs: [BuilderDebUBsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_ubsan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (ubsan)
REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=2
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1485,7 +1777,7 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1521,7 +1813,7 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1557,7 +1849,115 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestMsan3:
needs: [BuilderDebMsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (msan)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestMsan4:
needs: [BuilderDebMsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (msan)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=4
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestMsan5:
needs: [BuilderDebMsan]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_memory
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (msan)
REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=5
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1593,7 +1993,7 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=5
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1629,7 +2029,7 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=5
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -1665,7 +2065,79 @@ jobs:
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=5
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestDebug3:
needs: [BuilderDebDebug]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=5
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Functional test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
FunctionalStatelessTestDebug4:
needs: [BuilderDebDebug]
runs-on: [self-hosted, func-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/stateless_debug
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Stateless tests (debug)
REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
KILL_TIMEOUT=10800
RUN_BY_HASH_NUM=4
RUN_BY_HASH_TOTAL=5
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -2116,7 +2588,7 @@ jobs:
CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -2151,7 +2623,7 @@ jobs:
CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -2186,7 +2658,112 @@ jobs:
CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=3
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAsan3:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAsan4:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=4
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAsan5:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=5
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -2221,7 +2798,7 @@ jobs:
CHECK_NAME=Integration tests (tsan)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=4
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -2256,7 +2833,7 @@ jobs:
CHECK_NAME=Integration tests (tsan)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=4
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -2291,7 +2868,7 @@ jobs:
CHECK_NAME=Integration tests (tsan)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=4
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -2326,7 +2903,77 @@ jobs:
CHECK_NAME=Integration tests (tsan)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=4
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan4:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (tsan)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=4
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan5:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_tsan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (tsan)
REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse
RUN_BY_HASH_NUM=5
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -2361,7 +3008,7 @@ jobs:
CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=2
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -2396,7 +3043,77 @@ jobs:
CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=2
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsRelease2:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_release
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsRelease3:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_release
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (release)
REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v3
@ -3116,23 +3833,36 @@ jobs:
- FunctionalStatelessTestDebug0
- FunctionalStatelessTestDebug1
- FunctionalStatelessTestDebug2
- FunctionalStatelessTestDebug3
- FunctionalStatelessTestDebug4
- FunctionalStatelessTestRelease
- FunctionalStatelessTestReleaseDatabaseOrdinary
- FunctionalStatelessTestReleaseDatabaseReplicated0
- FunctionalStatelessTestReleaseDatabaseReplicated1
- FunctionalStatelessTestReleaseDatabaseReplicated2
- FunctionalStatelessTestReleaseDatabaseReplicated3
- FunctionalStatelessTestAarch64
- FunctionalStatelessTestAsan0
- FunctionalStatelessTestAsan1
- FunctionalStatelessTestAsan2
- FunctionalStatelessTestAsan3
- FunctionalStatelessTestTsan0
- FunctionalStatelessTestTsan1
- FunctionalStatelessTestTsan2
- FunctionalStatelessTestTsan3
- FunctionalStatelessTestTsan4
- FunctionalStatelessTestMsan0
- FunctionalStatelessTestMsan1
- FunctionalStatelessTestMsan2
- FunctionalStatelessTestUBsan
- FunctionalStatelessTestMsan3
- FunctionalStatelessTestMsan4
- FunctionalStatelessTestMsan5
- FunctionalStatelessTestUBsan0
- FunctionalStatelessTestUBsan1
- FunctionalStatefulTestDebug
- FunctionalStatefulTestRelease
- FunctionalStatelessTestReleaseS3
- FunctionalStatelessTestReleaseS3_0
- FunctionalStatelessTestReleaseS3_1
- FunctionalStatefulTestAarch64
- FunctionalStatefulTestAsan
- FunctionalStatefulTestTsan
@ -3146,12 +3876,19 @@ jobs:
- IntegrationTestsAsan0
- IntegrationTestsAsan1
- IntegrationTestsAsan2
- IntegrationTestsAsan3
- IntegrationTestsAsan4
- IntegrationTestsAsan5
- IntegrationTestsRelease0
- IntegrationTestsRelease1
- IntegrationTestsRelease2
- IntegrationTestsRelease3
- IntegrationTestsTsan0
- IntegrationTestsTsan1
- IntegrationTestsTsan2
- IntegrationTestsTsan3
- IntegrationTestsTsan4
- IntegrationTestsTsan5
- PerformanceComparisonX86-0
- PerformanceComparisonX86-1
- PerformanceComparisonX86-2

2
base/base/strong_typedef.h
View File

@ -35,7 +35,7 @@ public:
Self & operator=(T && rhs) { t = std::move(rhs); return *this;}
// NOLINTBEGIN(google-explicit-constructor)
operator const T & () const { return t; }
constexpr operator const T & () const { return t; }
operator T & () { return t; }
// NOLINTEND(google-explicit-constructor)

26
docs/changelogs/v22.12.6.22-stable.md Normal file
View File

@ -0,0 +1,26 @@
---
sidebar_position: 1
sidebar_label: 2023
---
# 2023 Changelog
### ClickHouse release v22.12.6.22-stable (10d87f90261) FIXME as compared to v22.12.5.34-stable (b82d6401ca1)
#### Bug Fix (user-visible misbehavior in an official stable release)
* Fix changing an expired role [#46772](https://github.com/ClickHouse/ClickHouse/pull/46772) ([Vitaly Baranov](https://github.com/vitlibar)).
* Fix bug in zero-copy replication disk choice during fetch [#47010](https://github.com/ClickHouse/ClickHouse/pull/47010) ([alesapin](https://github.com/alesapin)).
* Fix NOT_IMPLEMENTED error with CROSS JOIN and algorithm = auto [#47068](https://github.com/ClickHouse/ClickHouse/pull/47068) ([Vladimir C](https://github.com/vdimir)).
* Fix query parameters [#47488](https://github.com/ClickHouse/ClickHouse/pull/47488) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Hotfix for too verbose warnings in HTTP [#47903](https://github.com/ClickHouse/ClickHouse/pull/47903) ([Alexander Tokmakov](https://github.com/tavplubix)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Better error messages in ReplicatedMergeTreeAttachThread [#47454](https://github.com/ClickHouse/ClickHouse/pull/47454) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Add a fuse for backport branches w/o a created PR [#47760](https://github.com/ClickHouse/ClickHouse/pull/47760) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Only valid Reviews.STATES overwrite existing reviews [#47789](https://github.com/ClickHouse/ClickHouse/pull/47789) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Place short return before big block, improve logging [#47822](https://github.com/ClickHouse/ClickHouse/pull/47822) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Artifacts s3 prefix [#47945](https://github.com/ClickHouse/ClickHouse/pull/47945) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Fix tsan error lock-order-inversion [#47953](https://github.com/ClickHouse/ClickHouse/pull/47953) ([Kruglov Pavel](https://github.com/Avogar)).

29
docs/changelogs/v22.3.20.29-lts.md Normal file
View File

@ -0,0 +1,29 @@
---
sidebar_position: 1
sidebar_label: 2023
---
# 2023 Changelog
### ClickHouse release v22.3.20.29-lts (297b4dd5e55) FIXME as compared to v22.3.19.6-lts (467e0a7bd77)
#### Improvement
* Backported in [#46979](https://github.com/ClickHouse/ClickHouse/issues/46979): Apply `ALTER TABLE table_name ON CLUSTER cluster MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name'` to all replicas. Because `ALTER TABLE t MOVE` is not replicated. [#46402](https://github.com/ClickHouse/ClickHouse/pull/46402) ([lizhuoyu5](https://github.com/lzydmxy)).
#### Bug Fix (user-visible misbehavior in an official stable release)
* Fix incorrect alias recursion in QueryNormalizer [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)).
* Fix arithmetic operations in aggregate optimization [#46705](https://github.com/ClickHouse/ClickHouse/pull/46705) ([Duc Canh Le](https://github.com/canhld94)).
* Fix MSan report in `maxIntersections` function [#46847](https://github.com/ClickHouse/ClickHouse/pull/46847) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Fix wrong results of some LIKE searches when the LIKE pattern contains quoted non-quotable characters [#46875](https://github.com/ClickHouse/ClickHouse/pull/46875) ([Robert Schulze](https://github.com/rschu1ze)).
* Fix possible deadlock in QueryStatus [#47161](https://github.com/ClickHouse/ClickHouse/pull/47161) ([Kruglov Pavel](https://github.com/Avogar)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Add a fuse for backport branches w/o a created PR [#47760](https://github.com/ClickHouse/ClickHouse/pull/47760) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Only valid Reviews.STATES overwrite existing reviews [#47789](https://github.com/ClickHouse/ClickHouse/pull/47789) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Place short return before big block, improve logging [#47822](https://github.com/ClickHouse/ClickHouse/pull/47822) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Fix tsan error lock-order-inversion [#47953](https://github.com/ClickHouse/ClickHouse/pull/47953) ([Kruglov Pavel](https://github.com/Avogar)).

161
docs/en/getting-started/example-datasets/youtube-dislikes.md
View File

@ -67,7 +67,8 @@ CREATE TABLE youtube
(
`id` String,
`fetch_date` DateTime,
`upload_date` String,
`upload_date_str` String,
`upload_date` Date,
`title` String,
`uploader_id` String,
`uploader` String,
@ -87,7 +88,7 @@ CREATE TABLE youtube
`video_badges` String
)
ENGINE = MergeTree
ORDER BY (upload_date, uploader);
ORDER BY (uploader, upload_date);
```
3. The following command streams the records from the S3 files into the `youtube` table.
@ -101,8 +102,9 @@ INSERT INTO youtube
SETTINGS input_format_null_as_default = 1
SELECT
id,
parseDateTimeBestEffortUS(toString(fetch_date)) AS fetch_date,
upload_date,
parseDateTimeBestEffortUSOrZero(toString(fetch_date)) AS fetch_date,
upload_date AS upload_date_str,
toDate(parseDateTimeBestEffortUSOrZero(upload_date::String)) AS upload_date,
ifNull(title, '') AS title,
uploader_id,
ifNull(uploader, '') AS uploader,
@ -121,13 +123,26 @@ SELECT
ifNull(uploader_badges, '') AS uploader_badges,
ifNull(video_badges, '') AS video_badges
FROM s3Cluster(
'default',
'https://clickhouse-public-datasets.s3.amazonaws.com/youtube/original/files/*.zst',
'JSONLines'
);
'default',
'https://clickhouse-public-datasets.s3.amazonaws.com/youtube/original/files/*.zst',
'JSONLines'
)
SETTINGS
max_download_threads = 24,
max_insert_threads = 64,
max_insert_block_size = 100000000,
min_insert_block_size_rows = 100000000,
min_insert_block_size_bytes = 500000000;
```
4. Open a new tab in the SQL Console of ClickHouse Cloud (or a new `clickhouse-client` window) and watch the count increase. It will take a while to insert 4.56B rows, depending on your server resources. (Withtout any tweaking of settings, it takes about 4.5 hours.)
Some comments about our `INSERT` command:
- The `parseDateTimeBestEffortUSOrZero` function is handy when the incoming date fields may not be in the proper format. If `fetch_date` does not get parsed properly, it will be set to `0`
- The `upload_date` column contains valid dates, but it also contains strings like "4 hours ago" - which is certainly not a valid date. We decided to store the original value in `upload_date_str` and attempt to parse it with `toDate(parseDateTimeBestEffortUSOrZero(upload_date::String))`. If the parsing fails we just get `0`
- We used `ifNull` to avoid getting `NULL` values in our table. If an incoming value is `NULL`, the `ifNull` function is setting the value to an empty string
- It takes a long time to download the data, so we added a `SETTINGS` clause to spread out the work over more threads while making sure the block sizes stayed fairly large
4. Open a new tab in the SQL Console of ClickHouse Cloud (or a new `clickhouse-client` window) and watch the count increase. It will take a while to insert 4.56B rows, depending on your server resources. (Without any tweaking of settings, it takes about 4.5 hours.)
```sql
SELECT formatReadableQuantity(count())
@ -200,7 +215,7 @@ FROM youtube
WHERE (title ILIKE '%ClickHouse%') OR (description ILIKE '%ClickHouse%')
ORDER BY
like_count DESC,
view_count DESC
view_count DESC;
```
This query has to process every row, and also parse through two columns of strings. Even then, we get decent performance at 4.15M rows/second:
@ -224,7 +239,6 @@ The results look like:
When commenting is disabled, are people more likely to like or dislike to express their feelings about a video?
```sql
SELECT
concat('< ', formatReadableQuantity(view_range)) AS views,
@ -276,6 +290,127 @@ ORDER BY
Enabling comments seems to be correlated with a higher rate of engagement.
### How does the number of videos change over time - notable events?
```sql
SELECT
toStartOfMonth(toDateTime(upload_date)) AS month,
uniq(uploader_id) AS uploaders,
count() as num_videos,
sum(view_count) as view_count
FROM youtube
GROUP BY month
ORDER BY month ASC;
```
```response
┌──────month─┬─uploaders─┬─num_videos─┬───view_count─┐
│ 2005-04-01 │ 5 │ 6 │ 213597737 │
│ 2005-05-01 │ 6 │ 9 │ 2944005 │
│ 2005-06-01 │ 165 │ 351 │ 18624981 │
│ 2005-07-01 │ 395 │ 1168 │ 94164872 │
│ 2005-08-01 │ 1171 │ 3128 │ 124540774 │
│ 2005-09-01 │ 2418 │ 5206 │ 475536249 │
│ 2005-10-01 │ 6750 │ 13747 │ 737593613 │
│ 2005-11-01 │ 13706 │ 28078 │ 1896116976 │
│ 2005-12-01 │ 24756 │ 49885 │ 2478418930 │
│ 2006-01-01 │ 49992 │ 100447 │ 4532656581 │
│ 2006-02-01 │ 67882 │ 138485 │ 5677516317 │
│ 2006-03-01 │ 103358 │ 212237 │ 8430301366 │
│ 2006-04-01 │ 114615 │ 234174 │ 9980760440 │
│ 2006-05-01 │ 152682 │ 332076 │ 14129117212 │
│ 2006-06-01 │ 193962 │ 429538 │ 17014143263 │
│ 2006-07-01 │ 234401 │ 530311 │ 18721143410 │
│ 2006-08-01 │ 281280 │ 614128 │ 20473502342 │
│ 2006-09-01 │ 312434 │ 679906 │ 23158422265 │
│ 2006-10-01 │ 404873 │ 897590 │ 27357846117 │
```
A spike of uploaders [around covid is noticeable](https://www.theverge.com/2020/3/27/21197642/youtube-with-me-style-videos-views-coronavirus-cook-workout-study-home-beauty).
### More subtitiles over time and when
With advances in speech recognition, it’s easier than ever to create subtitles for video with youtube adding auto-captioning in late 2009 - was the jump then?
```sql
SELECT
toStartOfMonth(upload_date) AS month,
countIf(has_subtitles) / count() AS percent_subtitles,
percent_subtitles - any(percent_subtitles) OVER (
ORDER BY month ASC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING
) AS previous
FROM youtube
GROUP BY month
ORDER BY month ASC;
```
```response
┌──────month─┬───percent_subtitles─┬────────────────previous─┐
│ 2015-01-01 │ 0.2652653881082824 │ 0.2652653881082824 │
│ 2015-02-01 │ 0.3147556050309162 │ 0.049490216922633834 │
│ 2015-03-01 │ 0.32460464492371877 │ 0.009849039892802558 │
│ 2015-04-01 │ 0.33471963051468445 │ 0.010114985590965686 │
│ 2015-05-01 │ 0.3168087575501062 │ -0.017910872964578273 │
│ 2015-06-01 │ 0.3162609788438222 │ -0.0005477787062839745 │
│ 2015-07-01 │ 0.31828767677518033 │ 0.0020266979313581235 │
│ 2015-08-01 │ 0.3045551564286859 │ -0.013732520346494415 │
│ 2015-09-01 │ 0.311221133995152 │ 0.006665977566466086 │
│ 2015-10-01 │ 0.30574870926812175 │ -0.005472424727030245 │
│ 2015-11-01 │ 0.31125409712077234 │ 0.0055053878526505895 │
│ 2015-12-01 │ 0.3190967954651779 │ 0.007842698344405541 │
│ 2016-01-01 │ 0.32636021432496176 │ 0.007263418859783877 │
```
The data results show a spike in 2009. Apparently at that, time YouTube was removing their community captions feature, which allowed you to upload captions for other people's video.
This prompted a very successful campaign to have creators add captions to their videos for hard of hearing and deaf viewers.
### Top uploaders over time
```sql
WITH uploaders AS
(
SELECT uploader
FROM youtube
GROUP BY uploader
ORDER BY sum(view_count) DESC
LIMIT 10
)
SELECT
month,
uploader,
sum(view_count) AS total_views,
avg(dislike_count / like_count) AS like_to_dislike_ratio
FROM youtube
WHERE uploader IN (uploaders)
GROUP BY
toStartOfMonth(upload_date) AS month,
uploader
ORDER BY
month ASC,
total_views DESC;
```
```response
┌──────month─┬─uploader───────────────────┬─total_views─┬─like_to_dislike_ratio─┐
│ 1970-01-01 │ T-Series │ 10957099 │ 0.022784656361208206 │
│ 1970-01-01 │ Ryan's World │ 0 │ 0.003035559410234172 │
│ 1970-01-01 │ SET India │ 0 │ nan │
│ 2006-09-01 │ Cocomelon - Nursery Rhymes │ 256406497 │ 0.7005566715978622 │
│ 2007-06-01 │ Cocomelon - Nursery Rhymes │ 33641320 │ 0.7088650914344298 │
│ 2008-02-01 │ WWE │ 43733469 │ 0.07198856488734842 │
│ 2008-03-01 │ WWE │ 16514541 │ 0.1230603715431997 │
│ 2008-04-01 │ WWE │ 5907295 │ 0.2089399470159618 │
│ 2008-05-01 │ WWE │ 7779627 │ 0.09101676560436774 │
│ 2008-06-01 │ WWE │ 7018780 │ 0.0974184753155297 │
│ 2008-07-01 │ WWE │ 4686447 │ 0.1263845422065158 │
│ 2008-08-01 │ WWE │ 4514312 │ 0.08384574274791441 │
│ 2008-09-01 │ WWE │ 3717092 │ 0.07872802579349912 │
```
### How do like ratio changes as views go up?
```sql
@ -322,8 +457,6 @@ ORDER BY
│ < 10.00 billion │ false │ 1.77 │
│ < 10.00 billion │ true │ 19.5 │
└───────────────────┴─────────────────────┴────────────┘
20 rows in set. Elapsed: 63.664 sec. Processed 4.56 billion rows, 113.93 GB (71.59 million rows/s., 1.79 GB/s.)
```
### How are views distributed?
@ -359,6 +492,4 @@ ARRAY JOIN
│ 20th │ 16 │
│ 10th │ 6 │
└────────────┴─────────┘
12 rows in set. Elapsed: 1.864 sec. Processed 4.56 billion rows, 36.46 GB (2.45 billion rows/s., 19.56 GB/s.)
```

2
docs/en/operations/named-collections.md
View File

@ -244,10 +244,12 @@ Example of configuration:
<database>system</database>
<user>foo</user>
<password>secret</password>
<secure>1</secure>
</remote1>
</named_collections>
</clickhouse>
```
`secure` is not needed for connection because of `remoteSecure`, but it can be used for dictionaries.
### Example of using named collections with the `remote`/`remoteSecure` functions

2
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -1047,7 +1047,7 @@ Default value: `0`.
Sets the number of threads performing background merges and mutations for tables with MergeTree engines. This setting is also could be applied at server startup from the `default` profile configuration for backward compatibility at the ClickHouse server start. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance.
Before changing it, please also take a look at related MergeTree settings, such as `number_of_free_entries_in_pool_to_lower_max_size_of_merge` and `number_of_free_entries_in_pool_to_execute_mutation`.
Before changing it, please also take a look at related MergeTree settings, such as [number_of_free_entries_in_pool_to_lower_max_size_of_merge](../../operations/settings/merge-tree-settings.md#number-of-free-entries-in-pool-to-lower-max-size-of-merge) and [number_of_free_entries_in_pool_to_execute_mutation](../../operations/settings/merge-tree-settings.md#number-of-free-entries-in-pool-to-execute-mutation).
Possible values:

26
docs/en/operations/settings/merge-tree-settings.md
View File

@ -553,6 +553,32 @@ Default value: 8192
Merge reads rows from parts in blocks of `merge_max_block_size` rows, then merges and writes the result into a new part. The read block is placed in RAM, so `merge_max_block_size` affects the size of the RAM required for the merge. Thus, merges can consume a large amount of RAM for tables with very wide rows (if the average row size is 100kb, then when merging 10 parts, (100kb * 10 * 8192) = ~ 8GB of RAM). By decreasing `merge_max_block_size`, you can reduce the amount of RAM required for a merge but slow down a merge.
## number_of_free_entries_in_pool_to_lower_max_size_of_merge {#number-of-free-entries-in-pool-to-lower-max-size-of-merge}
When there is less than specified number of free entries in pool (or replicated queue), start to lower maximum size of merge to process (or to put in queue).
This is to allow small merges to process - not filling the pool with long running merges.
Possible values:
- Any positive integer.
Default value: 8
## number_of_free_entries_in_pool_to_execute_mutation {#number-of-free-entries-in-pool-to-execute-mutation}
When there is less than specified number of free entries in pool, do not execute part mutations.
This is to leave free threads for regular merges and avoid "Too many parts".
Possible values:
- Any positive integer.
Default value: 20
**Usage**
The value of the `number_of_free_entries_in_pool_to_execute_mutation` setting should be less than the value of the [background_pool_size](/docs/en/operations/server-configuration-parameters/settings#background_pool_size) * [background_pool_size](/docs/en/operations/server-configuration-parameters/settings#background_merges_mutations_concurrency_ratio). Otherwise, ClickHouse throws an exception.
## max_part_loading_threads {#max-part-loading-threads}
The maximum number of threads that read parts when ClickHouse starts.

6
docs/en/operations/tips.md
View File

@ -51,10 +51,14 @@ But for storing archives with rare queries, shelves will work.
## RAID {#raid}
When using HDD, you can combine their RAID-10, RAID-5, RAID-6 or RAID-50.
For Linux, software RAID is better (with `mdadm`). We do not recommend using LVM.
For Linux, software RAID is better (with `mdadm`).
When creating RAID-10, select the `far` layout.
If your budget allows, choose RAID-10.
LVM by itself (without RAID or `mdadm`) is ok, but making RAID with it or combining it with `mdadm` is a less explored option, and there will be more chances for mistakes
(selecting wrong chunk size; misalignment of chunks; choosing a wrong raid type; forgetting to cleanup disks). If you are confident
in using LVM, there is nothing against using it.
If you have more than 4 disks, use RAID-6 (preferred) or RAID-50, instead of RAID-5.
When using RAID-5, RAID-6 or RAID-50, always increase stripe_cache_size, since the default value is usually not the best choice.

8
docs/en/sql-reference/statements/system.md
View File

@ -283,10 +283,14 @@ SYSTEM START REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
Wait until a `ReplicatedMergeTree` table will be synced with other replicas in a cluster, but no more than `receive_timeout` seconds.
``` sql
SYSTEM SYNC REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_family_table_name [STRICT]
SYSTEM SYNC REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_family_table_name [STRICT | LIGHTWEIGHT | PULL]
```
After running this statement the `[db.]replicated_merge_tree_family_table_name` fetches commands from the common replicated log into its own replication queue, and then the query waits till the replica processes all of the fetched commands. If a `STRICT` modifier was specified then the query waits for the replication queue to become empty. The `STRICT` version may never succeed if new entries constantly appear in the replication queue.
After running this statement the `[db.]replicated_merge_tree_family_table_name` fetches commands from the common replicated log into its own replication queue, and then the query waits till the replica processes all of the fetched commands. The following modifiers are supported:
- If a `STRICT` modifier was specified then the query waits for the replication queue to become empty. The `STRICT` version may never succeed if new entries constantly appear in the replication queue.
- If a `LIGHTWEIGHT` modifier was specified then the query waits only for `GET_PART`, `ATTACH_PART`, `DROP_RANGE`, `REPLACE_RANGE` and `DROP_PART` entries to be processed.
- If a `PULL` modifier was specified then the query pulls new replication queue entries from ZooKeeper, but does not wait for anything to be processed.
### RESTART REPLICA

8
docs/ru/sql-reference/statements/system.md
View File

@ -272,10 +272,14 @@ SYSTEM START REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
Ждет когда таблица семейства `ReplicatedMergeTree` будет синхронизирована с другими репликами в кластере, но не более `receive_timeout` секунд:
``` sql
SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name [STRICT]
SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name [STRICT | LIGHTWEIGHT | PULL]
```
После выполнения этого запроса таблица `[db.]replicated_merge_tree_family_table_name` загружает команды из общего реплицированного лога в свою собственную очередь репликации. Затем запрос ждет, пока реплика не обработает все загруженные команды. Если указан модификатор `STRICT`, то запрос ждёт когда очередь репликации станет пустой. Строгий вариант запроса может никогда не завершиться успешно, если в очереди репликации постоянно появляются новые записи.
После выполнения этого запроса таблица `[db.]replicated_merge_tree_family_table_name` загружает команды из общего реплицированного лога в свою собственную очередь репликации. Затем запрос ждет, пока реплика не обработает все загруженные команды. Поддерживаются следующие модификаторы:
- Если указан модификатор `STRICT`, то запрос ждёт когда очередь репликации станет пустой. Строгий вариант запроса может никогда не завершиться успешно, если в очереди репликации постоянно появляются новые записи.
- Если указан модификатор `LIGHTWEIGHT`, то запрос ждёт когда будут обработаны записи `GET_PART`, `ATTACH_PART`, `DROP_RANGE`, `REPLACE_RANGE` и `DROP_PART`.
- Если указан модификатор `PULL`, то запрос только загружает записи очереди репликации из ZooKeeper и не ждёт выполнения чего-либо.
### RESTART REPLICA {#query_language-system-restart-replica}

2
docs/zh/sql-reference/statements/system.md
View File

@ -240,7 +240,7 @@ SYSTEM START REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
``` sql
SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name
SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name [STRICT | LIGHTWEIGHT | PULL]
```
### RESTART REPLICA {#query_language-system-restart-replica}

6
programs/server/MetricsTransmitter.cpp
View File

@ -87,7 +87,7 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
if (send_events)
{
for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
for (ProfileEvents::Event i = ProfileEvents::Event(0), end = ProfileEvents::end(); i < end; ++i)
{
const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed);
const auto counter_increment = counter - prev_counters[i];
@ -100,7 +100,7 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
if (send_events_cumulative)
{
for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
for (ProfileEvents::Event i = ProfileEvents::Event(0), end = ProfileEvents::end(); i < end; ++i)
{
const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed);
std::string key{ProfileEvents::getName(static_cast<ProfileEvents::Event>(i))};
@ -110,7 +110,7 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
if (send_metrics)
{
for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i)
for (CurrentMetrics::Metric i = CurrentMetrics::Metric(0), end = CurrentMetrics::end(); i < end; ++i)
{
const auto value = CurrentMetrics::values[i].load(std::memory_order_relaxed);

9
src/AggregateFunctions/AggregateFunctionGroupArray.h
View File

@ -288,7 +288,8 @@ public:
readVarUInt(size, buf);
if (unlikely(size > AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE);
if (limit_num_elems && unlikely(size > max_elems))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size, it should not exceed {}", max_elems);
@ -367,7 +368,8 @@ struct GroupArrayNodeBase
UInt64 size;
readVarUInt(size, buf);
if (unlikely(size > AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE);
Node * node = reinterpret_cast<Node *>(arena->alignedAlloc(sizeof(Node) + size, alignof(Node)));
node->size = size;
@ -621,7 +623,8 @@ public:
return;
if (unlikely(elems > AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ARRAY_SIZE);
if (limit_num_elems && unlikely(elems > max_elems))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size, it should not exceed {}", max_elems);

6
src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h
View File

@ -79,7 +79,8 @@ public:
{
length_to_resize = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), params[1]);
if (length_to_resize > AGGREGATE_FUNCTION_GROUP_ARRAY_INSERT_AT_MAX_SIZE)
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", AGGREGATE_FUNCTION_GROUP_ARRAY_INSERT_AT_MAX_SIZE);
}
}
@ -167,7 +168,8 @@ public:
readVarUInt(size, buf);
if (size > AGGREGATE_FUNCTION_GROUP_ARRAY_INSERT_AT_MAX_SIZE)
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", AGGREGATE_FUNCTION_GROUP_ARRAY_INSERT_AT_MAX_SIZE);
Array & arr = data(place).value;

3
src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h
View File

@ -144,7 +144,8 @@ public:
readVarUInt(size, buf);
if (unlikely(size > AGGREGATE_FUNCTION_MOVING_MAX_ARRAY_SIZE))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", AGGREGATE_FUNCTION_MOVING_MAX_ARRAY_SIZE);
if (size > 0)
{

3
src/AggregateFunctions/AggregateFunctionGroupBitmapData.h
View File

@ -127,7 +127,8 @@ public:
if (size == 0)
throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect size (0) in groupBitmap.");
if (size > max_size)
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size in groupBitmap.");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size in groupBitmap (maximum: {})", max_size);
/// TODO: this is unnecessary copying - it will be better to read and deserialize in one pass.
std::unique_ptr<char[]> buf(new char[size]);

3
src/AggregateFunctions/AggregateFunctionHistogram.h
View File

@ -294,7 +294,8 @@ public:
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too many bins");
static constexpr size_t max_size = 1_GiB;
if (size > max_size)
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size in histogram.");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size in histogram (maximum: {})", max_size);
buf.readStrict(reinterpret_cast<char *>(points), size * sizeof(WeightedValue));
}

2
src/AggregateFunctions/AggregateFunctionIntervalLengthSum.h
View File

@ -117,7 +117,7 @@ struct AggregateFunctionIntervalLengthSumData
readBinary(size, buf);
if (unlikely(size > MAX_ARRAY_SIZE))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size (maximum: {})", MAX_ARRAY_SIZE);
segments.clear();
segments.reserve(size);

3
src/AggregateFunctions/AggregateFunctionMaxIntersections.h
View File

@ -140,7 +140,8 @@ public:
readVarUInt(size, buf);
if (unlikely(size > AGGREGATE_FUNCTION_MAX_INTERSECTIONS_MAX_ARRAY_SIZE))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", AGGREGATE_FUNCTION_MAX_INTERSECTIONS_MAX_ARRAY_SIZE);
auto & value = this->data(place).value;

3
src/AggregateFunctions/AggregateFunctionSequenceNextNode.h
View File

@ -324,7 +324,8 @@ public:
return;
if (unlikely(size > max_node_size_deserialize))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", max_node_size_deserialize);
auto & value = data(place).value;

65
src/AggregateFunctions/AggregateFunctionSparkbar.h
View File

@ -1,5 +1,7 @@
#pragma once
#include <base/arithmeticOverflow.h>
#include <array>
#include <string_view>
#include <DataTypes/DataTypeString.h>
@ -43,7 +45,19 @@ struct AggregateFunctionSparkbarData
auto [it, inserted] = points.insert({x, y});
if (!inserted)
it->getMapped() += y;
{
if constexpr (std::is_floating_point_v<Y>)
{
it->getMapped() += y;
return it->getMapped();
}
else
{
Y res;
bool has_overfllow = common::addOverflow(it->getMapped(), y, res);
it->getMapped() = has_overfllow ? std::numeric_limits<Y>::max() : res;
}
}
return it->getMapped();
}
@ -117,6 +131,7 @@ class AggregateFunctionSparkbar final
{
private:
static constexpr size_t BAR_LEVELS = 8;
const size_t width = 0;
/// Range for x specified in parameters.
@ -126,8 +141,8 @@ private:
size_t updateFrame(ColumnString::Chars & frame, Y value) const
{
static constexpr std::array<std::string_view, 9> bars{" ", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"};
const auto & bar = (isNaN(value) || value < 1 || 8 < value) ? bars[0] : bars[static_cast<UInt8>(value)];
static constexpr std::array<std::string_view, BAR_LEVELS + 1> bars{" ", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"};
const auto & bar = (isNaN(value) || value < 1 || static_cast<Y>(BAR_LEVELS) < value) ? bars[0] : bars[static_cast<UInt8>(value)];
frame.insert(bar.begin(), bar.end());
return bar.size();
}
@ -161,7 +176,7 @@ private:
}
PaddedPODArray<Y> histogram(width, 0);
PaddedPODArray<UInt64> fhistogram(width, 0);
PaddedPODArray<UInt64> count_histogram(width, 0); /// The number of points in each bucket
for (const auto & point : data.points)
{
@ -176,22 +191,30 @@ private:
Float64 w = histogram.size();
size_t index = std::min<size_t>(static_cast<size_t>(w / delta * value), histogram.size() - 1);
if (std::numeric_limits<Y>::max() - histogram[index] > point.getMapped())
Y res;
bool has_overfllow = false;
if constexpr (std::is_floating_point_v<Y>)
res = histogram[index] + point.getMapped();
else
has_overfllow = common::addOverflow(histogram[index], point.getMapped(), res);
if (unlikely(has_overfllow))
{
histogram[index] += point.getMapped();
fhistogram[index] += 1;
/// In case of overflow, just saturate
/// Do not count new values, because we do not know how many of them were added
histogram[index] = std::numeric_limits<Y>::max();
}
else
{
/// In case of overflow, just saturate
histogram[index] = std::numeric_limits<Y>::max();
histogram[index] = res;
count_histogram[index] += 1;
}
}
for (size_t i = 0; i < histogram.size(); ++i)
{
if (fhistogram[i] > 0)
histogram[i] /= fhistogram[i];
if (count_histogram[i] > 0)
histogram[i] /= count_histogram[i];
}
Y y_max = 0;
@ -209,12 +232,30 @@ private:
return;
}
/// Scale the histogram to the range [0, BAR_LEVELS]
for (auto & y : histogram)
{
if (isNaN(y) || y <= 0)
{
y = 0;
continue;
}
constexpr auto levels_num = static_cast<Y>(BAR_LEVELS - 1);
if constexpr (std::is_floating_point_v<Y>)
{
y = y / (y_max / levels_num) + 1;
}
else
y = y * 7 / y_max + 1;
{
Y scaled;
bool has_overfllow = common::mulOverflow<Y>(y, levels_num, scaled);
if (has_overfllow)
y = y / (y_max / levels_num) + 1;
else
y = scaled / y_max + 1;
}
}
size_t sz = 0;

3
src/AggregateFunctions/QuantileExact.h
View File

@ -58,7 +58,8 @@ struct QuantileExactBase
size_t size = 0;
readVarUInt(size, buf);
if (unlikely(size > QUANTILE_EXACT_MAX_ARRAY_SIZE))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", QUANTILE_EXACT_MAX_ARRAY_SIZE);
array.resize(size);
buf.readStrict(reinterpret_cast<char *>(array.data()), size * sizeof(array[0]));
}

3
src/AggregateFunctions/ReservoirSampler.h
View File

@ -213,7 +213,8 @@ public:
size_t size = std::min(total_values, sample_count);
static constexpr size_t MAX_RESERVOIR_SIZE = 1_GiB;
if (unlikely(size > MAX_RESERVOIR_SIZE))
throw DB::Exception(DB::ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw DB::Exception(DB::ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", MAX_RESERVOIR_SIZE);
samples.resize(size);

3
src/AggregateFunctions/ReservoirSamplerDeterministic.h
View File

@ -166,7 +166,8 @@ public:
static constexpr size_t MAX_RESERVOIR_SIZE = 1_GiB;
if (unlikely(size > MAX_RESERVOIR_SIZE))
throw DB::Exception(DB::ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size");
throw DB::Exception(DB::ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", MAX_RESERVOIR_SIZE);
samples.resize(size);
for (size_t i = 0; i < size; ++i)

599
src/Analyzer/Passes/CNF.cpp Normal file
View File

@ -0,0 +1,599 @@
#include <Analyzer/Passes/CNF.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/ConstantNode.h>
#include <Interpreters/TreeCNFConverter.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
#include <Functions/FunctionFactory.h>
#include <Common/checkStackSize.h>
namespace DB
{
namespace ErrorCodes
{
extern const int TOO_MANY_TEMPORARY_COLUMNS;
}
namespace Analyzer
{
namespace
{
bool isLogicalFunction(const FunctionNode & function_node)
{
const std::string_view name = function_node.getFunctionName();
return name == "and" || name == "or" || name == "not";
}
template <typename... Args>
QueryTreeNodePtr createFunctionNode(const FunctionOverloadResolverPtr & function_resolver, Args &&... args)
{
auto function_node = std::make_shared<FunctionNode>(function_resolver->getName());
auto & new_arguments = function_node->getArguments().getNodes();
new_arguments.reserve(sizeof...(args));
(new_arguments.push_back(std::forward<Args>(args)), ...);
function_node->resolveAsFunction(function_resolver);
return function_node;
}
size_t countAtoms(const QueryTreeNodePtr & node)
{
checkStackSize();
const auto * function_node = node->as<FunctionNode>();
if (!function_node || !isLogicalFunction(*function_node))
return 1;
size_t atom_count = 0;
const auto & arguments = function_node->getArguments().getNodes();
for (const auto & argument : arguments)
atom_count += countAtoms(argument);
return atom_count;
}
class SplitMultiLogicVisitor
{
public:
explicit SplitMultiLogicVisitor(ContextPtr context)
: current_context(std::move(context))
{}
void visit(QueryTreeNodePtr & node)
{
checkStackSize();
auto * function_node = node->as<FunctionNode>();
if (!function_node || !isLogicalFunction(*function_node))
return;
const auto & name = function_node->getFunctionName();
if (name == "and" || name == "or")
{
auto function_resolver = FunctionFactory::instance().get(name, current_context);
const auto & arguments = function_node->getArguments().getNodes();
if (arguments.size() > 2)
{
QueryTreeNodePtr current = arguments[0];
for (size_t i = 1; i < arguments.size(); ++i)
current = createFunctionNode(function_resolver, std::move(current), arguments[i]);
auto & new_function_node = current->as<FunctionNode &>();
function_node->getArguments().getNodes() = std::move(new_function_node.getArguments().getNodes());
function_node->resolveAsFunction(function_resolver);
}
}
else
{
assert(name == "not");
}
auto & arguments = function_node->getArguments().getNodes();
for (auto & argument : arguments)
visit(argument);
}
private:
ContextPtr current_context;
};
class PushNotVisitor
{
public:
explicit PushNotVisitor(const ContextPtr & context)
: not_function_resolver(FunctionFactory::instance().get("not", context))
, or_function_resolver(FunctionFactory::instance().get("or", context))
, and_function_resolver(FunctionFactory::instance().get("and", context))
{}
void visit(QueryTreeNodePtr & node, bool add_negation)
{
checkStackSize();
auto * function_node = node->as<FunctionNode>();
if (!function_node || !isLogicalFunction(*function_node))
{
if (add_negation)
node = createFunctionNode(not_function_resolver, std::move(node));
return;
}
std::string_view function_name = function_node->getFunctionName();
if (function_name == "and" || function_name == "or")
{
if (add_negation)
{
if (function_name == "and")
function_node->resolveAsFunction(or_function_resolver);
else
function_node->resolveAsFunction(and_function_resolver);
}
auto & arguments = function_node->getArguments().getNodes();
for (auto & argument : arguments)
visit(argument, add_negation);
return;
}
assert(function_name == "not");
auto & arguments = function_node->getArguments().getNodes();
assert(arguments.size() == 1);
node = arguments[0];
visit(node, !add_negation);
}
private:
const FunctionOverloadResolverPtr not_function_resolver;
const FunctionOverloadResolverPtr or_function_resolver;
const FunctionOverloadResolverPtr and_function_resolver;
};
class PushOrVisitor
{
public:
PushOrVisitor(ContextPtr context, size_t max_atoms_, size_t num_atoms_)
: max_atoms(max_atoms_)
, num_atoms(num_atoms_)
, and_resolver(FunctionFactory::instance().get("and", context))
, or_resolver(FunctionFactory::instance().get("or", context))
{}
bool visit(QueryTreeNodePtr & node)
{
if (max_atoms && num_atoms > max_atoms)
return false;
checkStackSize();
auto * function_node = node->as<FunctionNode>();
if (!function_node)
return true;
std::string_view name = function_node->getFunctionName();
if (name == "or" || name == "and")
{
auto & arguments = function_node->getArguments().getNodes();
for (auto & argument : arguments)
visit(argument);
}
if (name == "or")
{
auto & arguments = function_node->getArguments().getNodes();
assert(arguments.size() == 2);
size_t and_node_id = arguments.size();
for (size_t i = 0; i < arguments.size(); ++i)
{
auto & argument = arguments[i];
if (auto * argument_function_node = argument->as<FunctionNode>();
argument_function_node && argument_function_node->getFunctionName() == "and")
and_node_id = i;
}
if (and_node_id == arguments.size())
return true;
auto & other_node = arguments[1 - and_node_id];
auto & and_function_arguments = arguments[and_node_id]->as<FunctionNode &>().getArguments().getNodes();
auto lhs = createFunctionNode(or_resolver, other_node->clone(), std::move(and_function_arguments[0]));
num_atoms += countAtoms(other_node);
auto rhs = createFunctionNode(or_resolver, std::move(other_node), std::move(and_function_arguments[1]));
node = createFunctionNode(and_resolver, std::move(lhs), std::move(rhs));
visit(node);
}
return true;
}
private:
size_t max_atoms;
size_t num_atoms;
const FunctionOverloadResolverPtr and_resolver;
const FunctionOverloadResolverPtr or_resolver;
};
class CollectGroupsVisitor
{
public:
void visit(QueryTreeNodePtr & node)
{
CNF::OrGroup or_group;
visitImpl(node, or_group);
if (!or_group.empty())
and_group.insert(std::move(or_group));
}
CNF::AndGroup and_group;
private:
void visitImpl(QueryTreeNodePtr & node, CNF::OrGroup & or_group)
{
checkStackSize();
auto * function_node = node->as<FunctionNode>();
if (!function_node || !isLogicalFunction(*function_node))
{
or_group.insert(CNF::AtomicFormula{false, std::move(node)});
return;
}
std::string_view name = function_node->getFunctionName();
if (name == "and")
{
auto & arguments = function_node->getArguments().getNodes();
for (auto & argument : arguments)
{
CNF::OrGroup argument_or_group;
visitImpl(argument, argument_or_group);
if (!argument_or_group.empty())
and_group.insert(std::move(argument_or_group));
}
}
else if (name == "or")
{
auto & arguments = function_node->getArguments().getNodes();
for (auto & argument : arguments)
visitImpl(argument, or_group);
}
else
{
assert(name == "not");
auto & arguments = function_node->getArguments().getNodes();
or_group.insert(CNF::AtomicFormula{true, std::move(arguments[0])});
}
}
};
std::optional<CNF::AtomicFormula> tryInvertFunction(
const CNF::AtomicFormula & atom, const ContextPtr & context, const std::unordered_map<std::string, std::string> & inverse_relations)
{
auto * function_node = atom.node_with_hash.node->as<FunctionNode>();
if (!function_node)
return std::nullopt;
if (auto it = inverse_relations.find(function_node->getFunctionName()); it != inverse_relations.end())
{
auto inverse_function_resolver = FunctionFactory::instance().get(it->second, context);
function_node->resolveAsFunction(inverse_function_resolver);
return CNF::AtomicFormula{!atom.negative, atom.node_with_hash.node};
}
return std::nullopt;
}
}
bool CNF::AtomicFormula::operator==(const AtomicFormula & rhs) const
{
return negative == rhs.negative && node_with_hash == rhs.node_with_hash;
}
bool CNF::AtomicFormula::operator<(const AtomicFormula & rhs) const
{
if (node_with_hash.hash > rhs.node_with_hash.hash)
return false;
return node_with_hash.hash < rhs.node_with_hash.hash || negative < rhs.negative;
}
std::string CNF::dump() const
{
WriteBufferFromOwnString res;
bool first = true;
for (const auto & group : statements)
{
if (!first)
res << " AND ";
first = false;
res << "(";
bool first_in_group = true;
for (const auto & atom : group)
{
if (!first_in_group)
res << " OR ";
first_in_group = false;
if (atom.negative)
res << " NOT ";
res << atom.node_with_hash.node->formatASTForErrorMessage();
}
res << ")";
}
return res.str();
}
CNF & CNF::transformGroups(std::function<OrGroup(const OrGroup &)> fn)
{
AndGroup result;
for (const auto & group : statements)
{
auto new_group = fn(group);
if (!new_group.empty())
result.insert(std::move(new_group));
}
statements = std::move(result);
return *this;
}
CNF & CNF::transformAtoms(std::function<AtomicFormula(const AtomicFormula &)> fn)
{
transformGroups([fn](const OrGroup & group)
{
OrGroup result;
for (const auto & atom : group)
{
auto new_atom = fn(atom);
if (new_atom.node_with_hash.node)
result.insert(std::move(new_atom));
}
return result;
});
return *this;
}
CNF & CNF::pushNotIntoFunctions(const ContextPtr & context)
{
transformAtoms([&](const AtomicFormula & atom)
{
return pushNotIntoFunction(atom, context);
});
return *this;
}
CNF::AtomicFormula CNF::pushNotIntoFunction(const AtomicFormula & atom, const ContextPtr & context)
{
if (!atom.negative)
return atom;
static const std::unordered_map<std::string, std::string> inverse_relations = {
{"equals", "notEquals"},
{"less", "greaterOrEquals"},
{"lessOrEquals", "greater"},
{"in", "notIn"},
{"like", "notLike"},
{"empty", "notEmpty"},
{"notEquals", "equals"},
{"greaterOrEquals", "less"},
{"greater", "lessOrEquals"},
{"notIn", "in"},
{"notLike", "like"},
{"notEmpty", "empty"},
};
if (auto inverted_atom = tryInvertFunction(atom, context, inverse_relations);
inverted_atom.has_value())
return std::move(*inverted_atom);
return atom;
}
CNF & CNF::pullNotOutFunctions(const ContextPtr & context)
{
transformAtoms([&](const AtomicFormula & atom)
{
static const std::unordered_map<std::string, std::string> inverse_relations = {
{"notEquals", "equals"},
{"greaterOrEquals", "less"},
{"greater", "lessOrEquals"},
{"notIn", "in"},
{"notLike", "like"},
{"notEmpty", "empty"},
};
if (auto inverted_atom = tryInvertFunction(atom, context, inverse_relations);
inverted_atom.has_value())
return std::move(*inverted_atom);
return atom;
});
return *this;
}
CNF & CNF::filterAlwaysTrueGroups(std::function<bool(const OrGroup &)> predicate)
{
AndGroup filtered;
for (const auto & or_group : statements)
{
if (predicate(or_group))
filtered.insert(or_group);
}
statements = std::move(filtered);
return *this;
}
CNF & CNF::filterAlwaysFalseAtoms(std::function<bool(const AtomicFormula &)> predicate)
{
AndGroup filtered;
for (const auto & or_group : statements)
{
OrGroup filtered_group;
for (const auto & atom : or_group)
{
if (predicate(atom))
filtered_group.insert(atom);
}
if (!filtered_group.empty())
filtered.insert(std::move(filtered_group));
else
{
filtered.clear();
filtered_group.insert(AtomicFormula{false, QueryTreeNodePtrWithHash{std::make_shared<ConstantNode>(static_cast<UInt8>(0))}});
filtered.insert(std::move(filtered_group));
break;
}
}
statements = std::move(filtered);
return *this;
}
CNF & CNF::reduce()
{
while (true)
{
AndGroup new_statements = reduceOnceCNFStatements(statements);
if (statements == new_statements)
{
statements = filterCNFSubsets(statements);
return *this;
}
else
statements = new_statements;
}
}
void CNF::appendGroup(const AndGroup & and_group)
{
for (const auto & or_group : and_group)
statements.emplace(or_group);
}
CNF::CNF(AndGroup statements_)
: statements(std::move(statements_))
{}
std::optional<CNF> CNF::tryBuildCNF(const QueryTreeNodePtr & node, ContextPtr context, size_t max_growth_multiplier)
{
auto node_cloned = node->clone();
size_t atom_count = countAtoms(node_cloned);
size_t max_atoms = max_growth_multiplier ? std::max(MAX_ATOMS_WITHOUT_CHECK, atom_count * max_growth_multiplier) : 0;
{
SplitMultiLogicVisitor visitor(context);
visitor.visit(node_cloned);
}
{
PushNotVisitor visitor(context);
visitor.visit(node_cloned, false);
}
if (PushOrVisitor visitor(context, max_atoms, atom_count);
!visitor.visit(node_cloned))
return std::nullopt;
CollectGroupsVisitor collect_visitor;
collect_visitor.visit(node_cloned);
if (collect_visitor.and_group.empty())
return std::nullopt;
return CNF{std::move(collect_visitor.and_group)};
}
CNF CNF::toCNF(const QueryTreeNodePtr & node, ContextPtr context, size_t max_growth_multiplier)
{
auto cnf = tryBuildCNF(node, context, max_growth_multiplier);
if (!cnf)
throw Exception(ErrorCodes::TOO_MANY_TEMPORARY_COLUMNS,
"Cannot convert expression '{}' to CNF, because it produces to many clauses."
"Size of boolean formula in CNF can be exponential of size of source formula.");
return *cnf;
}
QueryTreeNodePtr CNF::toQueryTree(ContextPtr context) const
{
if (statements.empty())
return nullptr;
QueryTreeNodes and_arguments;
and_arguments.reserve(statements.size());
auto not_resolver = FunctionFactory::instance().get("not", context);
auto or_resolver = FunctionFactory::instance().get("or", context);
auto and_resolver = FunctionFactory::instance().get("and", context);
const auto function_node_from_atom = [&](const auto & atom) -> QueryTreeNodePtr
{
auto cloned_node = atom.node_with_hash.node->clone();
if (atom.negative)
return createFunctionNode(not_resolver, std::move(cloned_node));
return std::move(cloned_node);
};
for (const auto & or_group : statements)
{
if (or_group.size() == 1)
{
const auto & atom = *or_group.begin();
and_arguments.push_back(function_node_from_atom(atom));
}
else
{
QueryTreeNodes or_arguments;
or_arguments.reserve(or_group.size());
for (const auto & atom : or_group)
or_arguments.push_back(function_node_from_atom(atom));
auto or_function = std::make_shared<FunctionNode>("or");
or_function->getArguments().getNodes() = std::move(or_arguments);
or_function->resolveAsFunction(or_resolver);
and_arguments.push_back(std::move(or_function));
}
}
if (and_arguments.size() == 1)
return std::move(and_arguments[0]);
auto and_function = std::make_shared<FunctionNode>("and");
and_function->getArguments().getNodes() = std::move(and_arguments);
and_function->resolveAsFunction(and_resolver);
return and_function;
}
}
}

67
src/Analyzer/Passes/CNF.h Normal file
View File

@ -0,0 +1,67 @@
#pragma once
#include <Analyzer/HashUtils.h>
#include <Analyzer/IQueryTreeNode.h>
#include <Common/SipHash.h>
#include <Interpreters/Context_fwd.h>
#include <unordered_set>
namespace DB::Analyzer
{
class CNF
{
public:
struct AtomicFormula
{
bool negative = false;
QueryTreeNodePtrWithHash node_with_hash;
bool operator==(const AtomicFormula & rhs) const;
bool operator<(const AtomicFormula & rhs) const;
};
// Different hash is generated for different order, so we use std::set
using OrGroup = std::set<AtomicFormula>;
using AndGroup = std::set<OrGroup>;
std::string dump() const;
static constexpr size_t DEFAULT_MAX_GROWTH_MULTIPLIER = 20;
static constexpr size_t MAX_ATOMS_WITHOUT_CHECK = 200;
CNF & transformAtoms(std::function<AtomicFormula(const AtomicFormula &)> fn);
CNF & transformGroups(std::function<OrGroup(const OrGroup &)> fn);
CNF & filterAlwaysTrueGroups(std::function<bool(const OrGroup &)> predicate);
CNF & filterAlwaysFalseAtoms(std::function<bool(const AtomicFormula &)> predicate);
CNF & reduce();
void appendGroup(const AndGroup & and_group);
/// Convert "NOT fn" to a single node representing inverse of "fn"
CNF & pushNotIntoFunctions(const ContextPtr & context);
CNF & pullNotOutFunctions(const ContextPtr & context);
static AtomicFormula pushNotIntoFunction(const AtomicFormula & atom, const ContextPtr & context);
explicit CNF(AndGroup statements_);
static std::optional<CNF> tryBuildCNF(const QueryTreeNodePtr & node, ContextPtr context, size_t max_growth_multiplier = DEFAULT_MAX_GROWTH_MULTIPLIER);
static CNF toCNF(const QueryTreeNodePtr & node, ContextPtr context, size_t max_growth_multiplier = DEFAULT_MAX_GROWTH_MULTIPLIER);
QueryTreeNodePtr toQueryTree(ContextPtr context) const;
const auto & getStatements() const
{
return statements;
}
private:
AndGroup statements;
};
}

733
src/Analyzer/Passes/ConvertQueryToCNFPass.cpp Normal file
View File

@ -0,0 +1,733 @@
#include <Analyzer/Passes/ConvertQueryToCNFPass.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/TableNode.h>
#include <Analyzer/ColumnNode.h>
#include <Analyzer/TableFunctionNode.h>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/Passes/CNF.h>
#include <Analyzer/Utils.h>
#include <Storages/IStorage.h>
#include <Functions/FunctionFactory.h>
#include "Analyzer/HashUtils.h"
#include "Analyzer/IQueryTreeNode.h"
#include "Interpreters/ComparisonGraph.h"
#include "base/types.h"
namespace DB
{
namespace
{
std::optional<Analyzer::CNF> tryConvertQueryToCNF(const QueryTreeNodePtr & node, const ContextPtr & context)
{
auto cnf_form = Analyzer::CNF::tryBuildCNF(node, context);
if (!cnf_form)
return std::nullopt;
cnf_form->pushNotIntoFunctions(context);
return cnf_form;
}
enum class MatchState : uint8_t
{
FULL_MATCH, /// a = b
PARTIAL_MATCH, /// a = not b
NONE,
};
MatchState match(const Analyzer::CNF::AtomicFormula & a, const Analyzer::CNF::AtomicFormula & b)
{
using enum MatchState;
if (a.node_with_hash != b.node_with_hash)
return NONE;
return a.negative == b.negative ? FULL_MATCH : PARTIAL_MATCH;
}
bool checkIfGroupAlwaysTrueFullMatch(const Analyzer::CNF::OrGroup & group, const ConstraintsDescription::QueryTreeData & query_tree_constraints)
{
/// We have constraints in CNF.
/// CNF is always true => Each OR group in CNF is always true.
/// So, we try to check whether we have al least one OR group from CNF as subset in our group.
/// If we've found one then our group is always true too.
const auto & constraints_data = query_tree_constraints.getConstraintData();
std::vector<size_t> found(constraints_data.size());
for (size_t i = 0; i < constraints_data.size(); ++i)
found[i] = constraints_data[i].size();
for (const auto & atom : group)
{
const auto constraint_atom_ids = query_tree_constraints.getAtomIds(atom.node_with_hash);
if (constraint_atom_ids)
{
const auto constraint_atoms = query_tree_constraints.getAtomsById(*constraint_atom_ids);
for (size_t i = 0; i < constraint_atoms.size(); ++i)
{
if (match(constraint_atoms[i], atom) == MatchState::FULL_MATCH)
{
if ((--found[(*constraint_atom_ids)[i].group_id]) == 0)
return true;
}
}
}
}
return false;
}
bool checkIfGroupAlwaysTrueGraph(const Analyzer::CNF::OrGroup & group, const ComparisonGraph<QueryTreeNodePtr> & graph)
{
/// We try to find at least one atom that is always true by using comparison graph.
for (const auto & atom : group)
{
const auto * function_node = atom.node_with_hash.node->as<FunctionNode>();
if (function_node)
{
const auto & arguments = function_node->getArguments().getNodes();
if (arguments.size() == 2)
{
const auto expected = ComparisonGraph<QueryTreeNodePtr>::atomToCompareResult(atom);
if (graph.isAlwaysCompare(expected, arguments[0], arguments[1]))
return true;
}
}
}
return false;
}
bool checkIfAtomAlwaysFalseFullMatch(const Analyzer::CNF::AtomicFormula & atom, const ConstraintsDescription::QueryTreeData & query_tree_constraints)
{
const auto constraint_atom_ids = query_tree_constraints.getAtomIds(atom.node_with_hash);
if (constraint_atom_ids)
{
for (const auto & constraint_atom : query_tree_constraints.getAtomsById(*constraint_atom_ids))
{
const auto match_result = match(constraint_atom, atom);
if (match_result == MatchState::PARTIAL_MATCH)
return true;
}
}
return false;
}
bool checkIfAtomAlwaysFalseGraph(const Analyzer::CNF::AtomicFormula & atom, const ComparisonGraph<QueryTreeNodePtr> & graph)
{
const auto * function_node = atom.node_with_hash.node->as<FunctionNode>();
if (!function_node)
return false;
const auto & arguments = function_node->getArguments().getNodes();
if (arguments.size() != 2)
return false;
/// TODO: special support for !=
const auto expected = ComparisonGraph<QueryTreeNodePtr>::atomToCompareResult(atom);
return !graph.isPossibleCompare(expected, arguments[0], arguments[1]);
}
void replaceToConstants(QueryTreeNodePtr & term, const ComparisonGraph<QueryTreeNodePtr> & graph)
{
const auto equal_constant = graph.getEqualConst(term);
if (equal_constant)
{
term = (*equal_constant)->clone();
return;
}
for (auto & child : term->getChildren())
{
if (child)
replaceToConstants(child, graph);
}
}
Analyzer::CNF::AtomicFormula replaceTermsToConstants(const Analyzer::CNF::AtomicFormula & atom, const ComparisonGraph<QueryTreeNodePtr> & graph)
{
auto node = atom.node_with_hash.node->clone();
replaceToConstants(node, graph);
return {atom.negative, std::move(node)};
}
StorageSnapshotPtr getStorageSnapshot(const QueryTreeNodePtr & node)
{
StorageSnapshotPtr storage_snapshot{nullptr};
if (auto * table_node = node->as<TableNode>())
return table_node->getStorageSnapshot();
else if (auto * table_function_node = node->as<TableFunctionNode>())
return table_function_node->getStorageSnapshot();
return nullptr;
}
bool onlyIndexColumns(const QueryTreeNodePtr & node, const std::unordered_set<std::string_view> & primary_key_set)
{
const auto * column_node = node->as<ColumnNode>();
/// TODO: verify that full name is correct here
if (column_node && !primary_key_set.contains(column_node->getColumnName()))
return false;
for (const auto & child : node->getChildren())
{
if (child && !onlyIndexColumns(child, primary_key_set))
return false;
}
return true;
}
bool onlyConstants(const QueryTreeNodePtr & node)
{
/// if it's only constant it will be already calculated
return node->as<ConstantNode>() != nullptr;
}
const std::unordered_map<std::string_view, ComparisonGraphCompareResult> & getRelationMap()
{
using enum ComparisonGraphCompareResult;
static const std::unordered_map<std::string_view, ComparisonGraphCompareResult> relations =
{
{"equals", EQUAL},
{"less", LESS},
{"lessOrEquals", LESS_OR_EQUAL},
{"greaterOrEquals", GREATER_OR_EQUAL},
{"greater", GREATER},
};
return relations;
}
const std::unordered_map<ComparisonGraphCompareResult, std::string> & getReverseRelationMap()
{
using enum ComparisonGraphCompareResult;
static const std::unordered_map<ComparisonGraphCompareResult, std::string> relations =
{
{EQUAL, "equals"},
{LESS, "less"},
{LESS_OR_EQUAL, "lessOrEquals"},
{GREATER_OR_EQUAL, "greaterOrEquals"},
{GREATER, "greater"},
};
return relations;
}
bool canBeSequence(const ComparisonGraphCompareResult left, const ComparisonGraphCompareResult right)
{
using enum ComparisonGraphCompareResult;
if (left == UNKNOWN || right == UNKNOWN || left == NOT_EQUAL || right == NOT_EQUAL)
return false;
if ((left == GREATER || left == GREATER_OR_EQUAL) && (right == LESS || right == LESS_OR_EQUAL))
return false;
if ((right == GREATER || right == GREATER_OR_EQUAL) && (left == LESS || left == LESS_OR_EQUAL))
return false;
return true;
}
ComparisonGraphCompareResult mostStrict(const ComparisonGraphCompareResult left, const ComparisonGraphCompareResult right)
{
using enum ComparisonGraphCompareResult;
if (left == LESS || left == GREATER)
return left;
if (right == LESS || right == GREATER)
return right;
if (left == LESS_OR_EQUAL || left == GREATER_OR_EQUAL)
return left;
if (right == LESS_OR_EQUAL || right == GREATER_OR_EQUAL)
return right;
if (left == EQUAL)
return left;
if (right == EQUAL)
return right;
return UNKNOWN;
}
/// Create OR-group for 'indexHint'.
/// Consider we have expression like A <op1> C, where C is constant.
/// Consider we have a constraint I <op2> A, where I depends only on columns from primary key.
/// Then if op1 and op2 forms a sequence of comparisons (e.g. A < C and I < A),
/// we can add to expression 'indexHint(I < A)' condition.
Analyzer::CNF::OrGroup createIndexHintGroup(
const Analyzer::CNF::OrGroup & group,
const ComparisonGraph<QueryTreeNodePtr> & graph,
const QueryTreeNodes & primary_key_only_nodes,
const ContextPtr & context)
{
Analyzer::CNF::OrGroup result;
for (const auto & atom : group)
{
const auto * function_node = atom.node_with_hash.node->as<FunctionNode>();
if (!function_node || !getRelationMap().contains(function_node->getFunctionName()))
continue;
const auto & arguments = function_node->getArguments().getNodes();
if (arguments.size() != 2)
continue;
auto check_and_insert = [&](const size_t index, const ComparisonGraphCompareResult expected_result)
{
if (!onlyConstants(arguments[1 - index]))
return false;
for (const auto & primary_key_node : primary_key_only_nodes)
{
ComparisonGraphCompareResult actual_result;
if (index == 0)
actual_result = graph.compare(primary_key_node, arguments[index]);
else
actual_result = graph.compare(arguments[index], primary_key_node);
if (canBeSequence(expected_result, actual_result))
{
auto helper_node = function_node->clone();
auto & helper_function_node = helper_node->as<FunctionNode &>();
helper_function_node.getArguments().getNodes()[index] = primary_key_node->clone();
auto reverse_function_name = getReverseRelationMap().at(mostStrict(expected_result, actual_result));
helper_function_node.resolveAsFunction(FunctionFactory::instance().get(reverse_function_name, context));
result.insert(Analyzer::CNF::AtomicFormula{atom.negative, std::move(helper_node)});
return true;
}
}
return false;
};
auto expected = getRelationMap().at(function_node->getFunctionName());
if (!check_and_insert(0, expected) && !check_and_insert(1, expected))
return {};
}
return result;
}
void addIndexConstraint(Analyzer::CNF & cnf, const QueryTreeNodes & table_expressions, const ContextPtr & context)
{
for (const auto & table_expression : table_expressions)
{
auto snapshot = getStorageSnapshot(table_expression);
if (!snapshot || !snapshot->metadata)
continue;
const auto primary_key = snapshot->metadata->getColumnsRequiredForPrimaryKey();
const std::unordered_set<std::string_view> primary_key_set(primary_key.begin(), primary_key.end());
const auto & query_tree_constraint = snapshot->metadata->getConstraints().getQueryTreeData(context, table_expression);
const auto & graph = query_tree_constraint.getGraph();
QueryTreeNodes primary_key_only_nodes;
for (const auto & vertex : graph.getVertices())
{
for (const auto & node : vertex)
{
if (onlyIndexColumns(node, primary_key_set))
primary_key_only_nodes.push_back(node);
}
}
Analyzer::CNF::AndGroup and_group;
const auto & statements = cnf.getStatements();
for (const auto & group : statements)
{
auto new_group = createIndexHintGroup(group, graph, primary_key_only_nodes, context);
if (!new_group.empty())
and_group.emplace(std::move(new_group));
}
if (!and_group.empty())
{
Analyzer::CNF::OrGroup new_group;
auto index_hint_node = std::make_shared<FunctionNode>("indexHint");
index_hint_node->getArguments().getNodes().push_back(Analyzer::CNF{std::move(and_group)}.toQueryTree(context));
index_hint_node->resolveAsFunction(FunctionFactory::instance().get("indexHint", context));
new_group.insert({false, QueryTreeNodePtrWithHash{std::move(index_hint_node)}});
cnf.appendGroup({new_group});
}
}
}
struct ColumnPrice
{
Int64 compressed_size{0};
Int64 uncompressed_size{0};
ColumnPrice(const Int64 compressed_size_, const Int64 uncompressed_size_)
: compressed_size(compressed_size_)
, uncompressed_size(uncompressed_size_)
{
}
bool operator<(const ColumnPrice & that) const
{
return std::tie(compressed_size, uncompressed_size) < std::tie(that.compressed_size, that.uncompressed_size);
}
ColumnPrice & operator+=(const ColumnPrice & that)
{
compressed_size += that.compressed_size;
uncompressed_size += that.uncompressed_size;
return *this;
}
ColumnPrice & operator-=(const ColumnPrice & that)
{
compressed_size -= that.compressed_size;
uncompressed_size -= that.uncompressed_size;
return *this;
}
};
using ColumnPriceByName = std::unordered_map<String, ColumnPrice>;
using ColumnPriceByQueryNode = QueryTreeNodePtrWithHashMap<ColumnPrice>;
class ComponentCollectorVisitor : public ConstInDepthQueryTreeVisitor<ComponentCollectorVisitor>
{
public:
ComponentCollectorVisitor(
std::set<UInt64> & components_,
QueryTreeNodePtrWithHashMap<UInt64> & query_node_to_component_,
const ComparisonGraph<QueryTreeNodePtr> & graph_)
: components(components_), query_node_to_component(query_node_to_component_), graph(graph_)
{}
void visitImpl(const QueryTreeNodePtr & node)
{
if (auto id = graph.getComponentId(node))
{
query_node_to_component.emplace(node, *id);
components.insert(*id);
}
}
private:
std::set<UInt64> & components;
QueryTreeNodePtrWithHashMap<UInt64> & query_node_to_component;
const ComparisonGraph<QueryTreeNodePtr> & graph;
};
class ColumnNameCollectorVisitor : public ConstInDepthQueryTreeVisitor<ColumnNameCollectorVisitor>
{
public:
ColumnNameCollectorVisitor(
std::unordered_set<std::string> & column_names_,
const QueryTreeNodePtrWithHashMap<UInt64> * query_node_to_component_)
: column_names(column_names_), query_node_to_component(query_node_to_component_)
{}
bool needChildVisit(const VisitQueryTreeNodeType & parent, const VisitQueryTreeNodeType &)
{
return !query_node_to_component || !query_node_to_component->contains(parent);
}
void visitImpl(const QueryTreeNodePtr & node)
{
if (query_node_to_component && query_node_to_component->contains(node))
return;
if (const auto * column_node = node->as<ColumnNode>())
column_names.insert(column_node->getColumnName());
}
private:
std::unordered_set<std::string> & column_names;
const QueryTreeNodePtrWithHashMap<UInt64> * query_node_to_component;
};
class SubstituteColumnVisitor : public InDepthQueryTreeVisitor<SubstituteColumnVisitor>
{
public:
SubstituteColumnVisitor(
const QueryTreeNodePtrWithHashMap<UInt64> & query_node_to_component_,
const std::unordered_map<UInt64, QueryTreeNodePtr> & id_to_query_node_map_,
ContextPtr context_)
: query_node_to_component(query_node_to_component_), id_to_query_node_map(id_to_query_node_map_), context(std::move(context_))
{}
void visitImpl(QueryTreeNodePtr & node)
{
auto component_id_it = query_node_to_component.find(node);
if (component_id_it == query_node_to_component.end())
return;
const auto component_id = component_id_it->second;
auto new_node = id_to_query_node_map.at(component_id)->clone();
if (!node->getResultType()->equals(*new_node->getResultType()))
{
node = buildCastFunction(new_node, node->getResultType(), context);
return;
}
node = std::move(new_node);
}
private:
const QueryTreeNodePtrWithHashMap<UInt64> & query_node_to_component;
const std::unordered_map<UInt64, QueryTreeNodePtr> & id_to_query_node_map;
ContextPtr context;
};
ColumnPrice calculatePrice(
const ColumnPriceByName & column_prices,
const std::unordered_set<std::string> & column_names)
{
ColumnPrice result(0, 0);
for (const auto & column : column_names)
{
if (auto it = column_prices.find(column); it != column_prices.end())
result += it->second;
}
return result;
}
void bruteForce(
const ComparisonGraph<QueryTreeNodePtr> & graph,
const std::vector<UInt64> & components,
size_t current_component,
const ColumnPriceByName & column_prices,
ColumnPrice current_price,
std::vector<QueryTreeNodePtr> & expressions_stack,
ColumnPrice & min_price,
std::vector<QueryTreeNodePtr> & min_expressions)
{
if (current_component == components.size())
{
if (current_price < min_price)
{
min_price = current_price;
min_expressions = expressions_stack;
}
return;
}
for (const auto & node : graph.getComponent(components[current_component]))
{
std::unordered_set<std::string> column_names;
ColumnNameCollectorVisitor column_name_collector{column_names, nullptr};
column_name_collector.visit(node);
ColumnPrice expression_price = calculatePrice(column_prices, column_names);
expressions_stack.push_back(node);
current_price += expression_price;
ColumnPriceByName new_prices(column_prices);
for (const auto & column : column_names)
new_prices.insert_or_assign(column, ColumnPrice(0, 0));
bruteForce(graph,
components,
current_component + 1,
new_prices,
current_price,
expressions_stack,
min_price,
min_expressions);
current_price -= expression_price;
expressions_stack.pop_back();
}
}
void substituteColumns(QueryNode & query_node, const QueryTreeNodes & table_expressions, const ContextPtr & context)
{
static constexpr UInt64 COLUMN_PENALTY = 10 * 1024 * 1024;
static constexpr Int64 INDEX_PRICE = -1'000'000'000'000'000'000;
for (const auto & table_expression : table_expressions)
{
auto snapshot = getStorageSnapshot(table_expression);
if (!snapshot || !snapshot->metadata)
continue;
const auto column_sizes = snapshot->storage.getColumnSizes();
if (column_sizes.empty())
return;
auto query_tree_constraint = snapshot->metadata->getConstraints().getQueryTreeData(context, table_expression);
const auto & graph = query_tree_constraint.getGraph();
auto run_for_all = [&](const auto function)
{
function(query_node.getProjectionNode());
if (query_node.hasWhere())
function(query_node.getWhere());
if (query_node.hasPrewhere())
function(query_node.getPrewhere());
if (query_node.hasHaving())
function(query_node.getHaving());
};
std::set<UInt64> components;
QueryTreeNodePtrWithHashMap<UInt64> query_node_to_component;
std::unordered_set<std::string> column_names;
run_for_all([&](QueryTreeNodePtr & node)
{
ComponentCollectorVisitor component_collector{components, query_node_to_component, graph};
component_collector.visit(node);
ColumnNameCollectorVisitor column_name_collector{column_names, &query_node_to_component};
column_name_collector.visit(node);
});
ColumnPriceByName column_prices;
const auto primary_key = snapshot->metadata->getColumnsRequiredForPrimaryKey();
for (const auto & [column_name, column_size] : column_sizes)
column_prices.insert_or_assign(column_name, ColumnPrice(column_size.data_compressed + COLUMN_PENALTY, column_size.data_uncompressed));
for (const auto & column_name : primary_key)
column_prices.insert_or_assign(column_name, ColumnPrice(INDEX_PRICE, INDEX_PRICE));
for (const auto & column_name : column_names)
column_prices.insert_or_assign(column_name, ColumnPrice(0, 0));
std::unordered_map<UInt64, QueryTreeNodePtr> id_to_query_node_map;
std::vector<UInt64> components_list;
for (const auto component_id : components)
{
auto component = graph.getComponent(component_id);
if (component.size() == 1)
id_to_query_node_map[component_id] = component.front();
else
components_list.push_back(component_id);
}
std::vector<QueryTreeNodePtr> expressions_stack;
ColumnPrice min_price(std::numeric_limits<Int64>::max(), std::numeric_limits<Int64>::max());
std::vector<QueryTreeNodePtr> min_expressions;
bruteForce(graph,
components_list,
0,
column_prices,
ColumnPrice(0, 0),
expressions_stack,
min_price,
min_expressions);
for (size_t i = 0; i < components_list.size(); ++i)
id_to_query_node_map[components_list[i]] = min_expressions[i];
SubstituteColumnVisitor substitute_column{query_node_to_component, id_to_query_node_map, context};
run_for_all([&](QueryTreeNodePtr & node)
{
substitute_column.visit(node);
});
}
}
void optimizeWithConstraints(Analyzer::CNF & cnf, const QueryTreeNodes & table_expressions, const ContextPtr & context)
{
cnf.pullNotOutFunctions(context);
for (const auto & table_expression : table_expressions)
{
auto snapshot = getStorageSnapshot(table_expression);
if (!snapshot || !snapshot->metadata)
continue;
const auto & constraints = snapshot->metadata->getConstraints();
const auto & query_tree_constraints = constraints.getQueryTreeData(context, table_expression);
const auto & compare_graph = query_tree_constraints.getGraph();
cnf.filterAlwaysTrueGroups([&](const auto & group)
{
/// remove always true groups from CNF
return !checkIfGroupAlwaysTrueFullMatch(group, query_tree_constraints) && !checkIfGroupAlwaysTrueGraph(group, compare_graph);
})
.filterAlwaysFalseAtoms([&](const Analyzer::CNF::AtomicFormula & atom)
{
/// remove always false atoms from CNF
return !checkIfAtomAlwaysFalseFullMatch(atom, query_tree_constraints) && !checkIfAtomAlwaysFalseGraph(atom, compare_graph);
})
.transformAtoms([&](const auto & atom)
{
return replaceTermsToConstants(atom, compare_graph);
})
.reduce();
}
cnf.pushNotIntoFunctions(context);
const auto & settings = context->getSettingsRef();
if (settings.optimize_append_index)
addIndexConstraint(cnf, table_expressions, context);
}
void optimizeNode(QueryTreeNodePtr & node, const QueryTreeNodes & table_expressions, const ContextPtr & context)
{
const auto & settings = context->getSettingsRef();
auto cnf = tryConvertQueryToCNF(node, context);
if (!cnf)
return;
if (settings.optimize_using_constraints)
optimizeWithConstraints(*cnf, table_expressions, context);
auto new_node = cnf->toQueryTree(context);
node = std::move(new_node);
}
class ConvertQueryToCNFVisitor : public InDepthQueryTreeVisitorWithContext<ConvertQueryToCNFVisitor>
{
public:
using Base = InDepthQueryTreeVisitorWithContext<ConvertQueryToCNFVisitor>;
using Base::Base;
void visitImpl(QueryTreeNodePtr & node)
{
auto * query_node = node->as<QueryNode>();
if (!query_node)
return;
auto table_expressions = extractTableExpressions(query_node->getJoinTree());
const auto & context = getContext();
const auto & settings = context->getSettingsRef();
bool has_filter = false;
const auto optimize_filter = [&](QueryTreeNodePtr & filter_node)
{
if (filter_node == nullptr)
return;
optimizeNode(filter_node, table_expressions, context);
has_filter = true;
};
optimize_filter(query_node->getWhere());
optimize_filter(query_node->getPrewhere());
optimize_filter(query_node->getHaving());
if (has_filter && settings.optimize_substitute_columns)
substituteColumns(*query_node, table_expressions, context);
}
};
}
void ConvertLogicalExpressionToCNFPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
{
const auto & settings = context->getSettingsRef();
if (!settings.convert_query_to_cnf)
return;
ConvertQueryToCNFVisitor visitor(std::move(context));
visitor.visit(query_tree_node);
}
}

18
src/Analyzer/Passes/ConvertQueryToCNFPass.h Normal file
View File

@ -0,0 +1,18 @@
#pragma once
#include <Analyzer/IQueryTreePass.h>
namespace DB
{
class ConvertLogicalExpressionToCNFPass final : public IQueryTreePass
{
public:
String getName() override { return "ConvertLogicalExpressionToCNFPass"; }
String getDescription() override { return "Convert logical expression to CNF and apply optimizations using constraints"; }
void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
};
}

6
src/Analyzer/QueryTreePassManager.cpp
View File

@ -41,7 +41,7 @@
#include <Analyzer/Passes/LogicalExpressionOptimizerPass.h>
#include <Analyzer/Passes/CrossToInnerJoinPass.h>
#include <Analyzer/Passes/ShardNumColumnToFunctionPass.h>
#include <Analyzer/Passes/ConvertQueryToCNFPass.h>
namespace DB
{
@ -148,8 +148,6 @@ private:
/** ClickHouse query tree pass manager.
*
* TODO: Support setting convert_query_to_cnf.
* TODO: Support setting optimize_using_constraints.
* TODO: Support setting optimize_substitute_columns.
* TODO: Support GROUP BY injective function elimination.
* TODO: Support setting optimize_move_functions_out_of_any.
@ -235,6 +233,8 @@ void addQueryTreePasses(QueryTreePassManager & manager)
manager.addPass(std::make_unique<QueryAnalysisPass>());
manager.addPass(std::make_unique<FunctionToSubcolumnsPass>());
manager.addPass(std::make_unique<ConvertLogicalExpressionToCNFPass>());
manager.addPass(std::make_unique<CountDistinctPass>());
manager.addPass(std::make_unique<RewriteAggregateFunctionWithIfPass>());
manager.addPass(std::make_unique<SumIfToCountIfPass>());

6
src/Common/CurrentMetrics.cpp
View File

@ -126,6 +126,8 @@
M(DDLWorkerThreadsActive, "Number of threads in the DDLWORKER thread pool for ON CLUSTER queries running a task.") \
M(StorageDistributedThreads, "Number of threads in the StorageDistributed thread pool.") \
M(StorageDistributedThreadsActive, "Number of threads in the StorageDistributed thread pool running a task.") \
M(DistributedInsertThreads, "Number of threads used for INSERT into Distributed.") \
M(DistributedInsertThreadsActive, "Number of threads used for INSERT into Distributed running a task.") \
M(StorageS3Threads, "Number of threads in the StorageS3 thread pool.") \
M(StorageS3ThreadsActive, "Number of threads in the StorageS3 thread pool running a task.") \
M(MergeTreePartsLoaderThreads, "Number of threads in the MergeTree parts loader thread pool.") \
@ -184,10 +186,10 @@
namespace CurrentMetrics
{
#define M(NAME, DOCUMENTATION) extern const Metric NAME = __COUNTER__;
#define M(NAME, DOCUMENTATION) extern const Metric NAME = Metric(__COUNTER__);
APPLY_FOR_METRICS(M)
#undef M
constexpr Metric END = __COUNTER__;
constexpr Metric END = Metric(__COUNTER__);
std::atomic<Value> values[END] {}; /// Global variable, initialized by zeros.

3
src/Common/CurrentMetrics.h
View File

@ -6,6 +6,7 @@
#include <atomic>
#include <cassert>
#include <base/types.h>
#include <base/strong_typedef.h>
/** Allows to count number of simultaneously happening processes or current value of some metric.
* - for high-level profiling.
@ -22,7 +23,7 @@
namespace CurrentMetrics
{
/// Metric identifier (index in array).
using Metric = size_t;
using Metric = StrongTypedef<size_t, struct MetricTag>;
using Value = DB::Int64;
/// Get name of metric by identifier. Returns statically allocated string.

10
src/Common/ProfileEvents.cpp
View File

@ -497,10 +497,10 @@ The server successfully detected this situation and will download merged part fr
namespace ProfileEvents
{
#define M(NAME, DOCUMENTATION) extern const Event NAME = __COUNTER__;
#define M(NAME, DOCUMENTATION) extern const Event NAME = Event(__COUNTER__);
APPLY_FOR_EVENTS(M)
#undef M
constexpr Event END = __COUNTER__;
constexpr Event END = Event(__COUNTER__);
/// Global variable, initialized by zeros.
Counter global_counters_array[END] {};
@ -522,7 +522,7 @@ void Counters::resetCounters()
{
if (counters)
{
for (Event i = 0; i < num_counters; ++i)
for (Event i = Event(0); i < num_counters; ++i)
counters[i].store(0, std::memory_order_relaxed);
}
}
@ -540,7 +540,7 @@ Counters::Snapshot::Snapshot()
Counters::Snapshot Counters::getPartiallyAtomicSnapshot() const
{
Snapshot res;
for (Event i = 0; i < num_counters; ++i)
for (Event i = Event(0); i < num_counters; ++i)
res.counters_holder[i] = counters[i].load(std::memory_order_relaxed);
return res;
}
@ -616,7 +616,7 @@ CountersIncrement::CountersIncrement(Counters::Snapshot const & snapshot)
CountersIncrement::CountersIncrement(Counters::Snapshot const & after, Counters::Snapshot const & before)
{
init();
for (Event i = 0; i < Counters::num_counters; ++i)
for (Event i = Event(0); i < Counters::num_counters; ++i)
increment_holder[i] = static_cast<Increment>(after[i]) - static_cast<Increment>(before[i]);
}

5
src/Common/ProfileEvents.h
View File

@ -1,7 +1,8 @@
#pragma once
#include <Common/VariableContext.h>
#include "base/types.h"
#include <base/types.h>
#include <base/strong_typedef.h>
#include <atomic>
#include <memory>
#include <cstddef>
@ -14,7 +15,7 @@
namespace ProfileEvents
{
/// Event identifier (index in array).
using Event = size_t;
using Event = StrongTypedef<size_t, struct EventTag>;
using Count = size_t;
using Increment = Int64;
using Counter = std::atomic<Count>;

4
src/Common/StatusInfo.cpp
View File

@ -8,10 +8,10 @@
namespace CurrentStatusInfo
{
#define M(NAME, DOCUMENTATION, ENUM) extern const Status NAME = __COUNTER__;
#define M(NAME, DOCUMENTATION, ENUM) extern const Status NAME = Status(__COUNTER__);
APPLY_FOR_STATUS(M)
#undef M
constexpr Status END = __COUNTER__;
constexpr Status END = Status(__COUNTER__);
std::mutex locks[END] {};
std::unordered_map<String, Int8> values[END] {};

3
src/Common/StatusInfo.h
View File

@ -6,13 +6,14 @@
#include <atomic>
#include <vector>
#include <base/types.h>
#include <base/strong_typedef.h>
#include <mutex>
#include <unordered_map>
namespace CurrentStatusInfo
{
using Status = size_t;
using Status = StrongTypedef<size_t, struct StatusTag>;
using Key = std::string;
const char * getName(Status event);

9
src/Common/tests/gtest_thread_pool_limit.cpp
View File

@ -1,16 +1,23 @@
#include <atomic>
#include <iostream>
#include <Common/ThreadPool.h>
#include <Common/CurrentMetrics.h>
#include <gtest/gtest.h>
namespace CurrentMetrics
{
extern const Metric LocalThread;
extern const Metric LocalThreadActive;
}
/// Test for thread self-removal when number of free threads in pool is too large.
/// Just checks that nothing weird happens.
template <typename Pool>
int test()
{
Pool pool(10, 2, 10);
Pool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 10, 2, 10);
std::atomic<int> counter{0};
for (size_t i = 0; i < 10; ++i)

7
src/DataTypes/DataTypeDecimalBase.h
View File

@ -71,9 +71,12 @@ public:
scale(scale_)
{
if (unlikely(precision < 1 || precision > maxPrecision()))
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Precision {} is out of bounds", std::to_string(precision));
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND,
"Precision {} is out of bounds (precision range: [1, {}])",
std::to_string(precision), maxPrecision());
if (unlikely(scale > maxPrecision()))
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds", std::to_string(scale));
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds (max scale: {})",
std::to_string(scale), maxPrecision());
}
TypeIndex getTypeId() const override { return TypeToTypeIndex<T>; }

8
src/DataTypes/DataTypesDecimal.h
View File

@ -116,7 +116,8 @@ inline ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & v
if (common::mulOverflow(static_cast<MaxNativeType>(value.value), converted_value, converted_value))
{
if constexpr (throw_exception)
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow", std::string(ToDataType::family_name));
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow while multiplying {} by scale {}",
std::string(ToDataType::family_name), toString(value.value), toString(converted_value));
else
return ReturnType(false);
}
@ -136,7 +137,10 @@ inline ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & v
converted_value > std::numeric_limits<typename ToFieldType::NativeType>::max())
{
if constexpr (throw_exception)
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow", std::string(ToDataType::family_name));
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow: {} is not in range ({}, {})",
std::string(ToDataType::family_name), toString(converted_value),
toString(std::numeric_limits<typename ToFieldType::NativeType>::min()),
toString(std::numeric_limits<typename ToFieldType::NativeType>::max()));
else
return ReturnType(false);
}

2
src/Databases/DatabaseReplicated.cpp
View File

@ -661,7 +661,7 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex
String node_path = ddl_worker->tryEnqueueAndExecuteEntry(entry, query_context);
Strings hosts_to_wait = getZooKeeper()->getChildren(zookeeper_path + "/replicas");
return getDistributedDDLStatus(node_path, entry, query_context, hosts_to_wait);
return getDistributedDDLStatus(node_path, entry, query_context, &hosts_to_wait);
}
static UUID getTableUUIDIfReplicated(const String & metadata, ContextPtr context)

113
src/Dictionaries/ClickHouseDictionarySource.cpp
View File

@ -7,10 +7,11 @@
#include <Interpreters/ExpressionActions.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <QueryPipeline/QueryPipelineBuilder.h>
#include <Storages/ExternalDataSourceConfiguration.h>
#include <Storages/checkAndGetLiteralArgument.h>
#include <IO/ConnectionTimeouts.h>
#include <Interpreters/Session.h>
#include <Interpreters/executeQuery.h>
#include <Storages/NamedCollectionsHelpers.h>
#include <Common/isLocalAddress.h>
#include <Common/logger_useful.h>
#include "DictionarySourceFactory.h"
@ -28,10 +29,6 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
static const std::unordered_set<std::string_view> dictionary_allowed_keys = {
"host", "port", "user", "password", "quota_key", "db", "database", "table",
"update_field", "update_lag", "invalidate_query", "query", "where", "name", "secure"};
namespace
{
constexpr size_t MAX_CONNECTIONS = 16;
@ -213,63 +210,75 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory)
const std::string & config_prefix,
Block & sample_block,
ContextPtr global_context,
const std::string & default_database [[maybe_unused]],
const std::string & default_database,
bool created_from_ddl) -> DictionarySourcePtr
{
bool secure = config.getBool(config_prefix + ".secure", false);
UInt16 default_port = getPortFromContext(global_context, secure);
using Configuration = ClickHouseDictionarySource::Configuration;
std::optional<Configuration> configuration;
std::string settings_config_prefix = config_prefix + ".clickhouse";
std::string host = config.getString(settings_config_prefix + ".host", "localhost");
std::string user = config.getString(settings_config_prefix + ".user", "default");
std::string password = config.getString(settings_config_prefix + ".password", "");
std::string quota_key = config.getString(settings_config_prefix + ".quota_key", "");
std::string db = config.getString(settings_config_prefix + ".db", default_database);
std::string table = config.getString(settings_config_prefix + ".table", "");
UInt16 port = static_cast<UInt16>(config.getUInt(settings_config_prefix + ".port", default_port));
auto has_config_key = [](const String & key) { return dictionary_allowed_keys.contains(key); };
auto named_collection = created_from_ddl
? getExternalDataSourceConfiguration(config, settings_config_prefix, global_context, has_config_key)
: std::nullopt;
auto named_collection = created_from_ddl ? tryGetNamedCollectionWithOverrides(config, settings_config_prefix) : nullptr;
if (named_collection)
{
const auto & configuration = named_collection->configuration;
host = configuration.host;
user = configuration.username;
password = configuration.password;
quota_key = configuration.quota_key;
db = configuration.database;
table = configuration.table;
port = configuration.port;
validateNamedCollection(
*named_collection, {}, ValidateKeysMultiset<ExternalDatabaseEqualKeysSet>{
"secure", "host", "hostnmae", "port", "user", "username", "password", "quota_key", "name",
"db", "database", "table","query", "where", "invalidate_query", "update_field", "update_lag"});
const auto secure = named_collection->getOrDefault("secure", false);
const auto default_port = getPortFromContext(global_context, secure);
const auto host = named_collection->getAnyOrDefault<String>({"host", "hostname"}, "localhost");
const auto port = static_cast<UInt16>(named_collection->getOrDefault<UInt64>("port", default_port));
configuration.emplace(Configuration{
.host = host,
.user = named_collection->getAnyOrDefault<String>({"user", "username"}, "default"),
.password = named_collection->getOrDefault<String>("password", ""),
.quota_key = named_collection->getOrDefault<String>("quota_key", ""),
.db = named_collection->getAnyOrDefault<String>({"db", "database"}, default_database),
.table = named_collection->getOrDefault<String>("table", ""),
.query = named_collection->getOrDefault<String>("query", ""),
.where = named_collection->getOrDefault<String>("where", ""),
.invalidate_query = named_collection->getOrDefault<String>("invalidate_query", ""),
.update_field = named_collection->getOrDefault<String>("update_field", ""),
.update_lag = named_collection->getOrDefault<UInt64>("update_lag", 1),
.port = port,
.is_local = isLocalAddress({host, port}, default_port),
.secure = secure,
});
}
else
{
const auto secure = config.getBool(settings_config_prefix + ".secure", false);
const auto default_port = getPortFromContext(global_context, secure);
const auto host = config.getString(settings_config_prefix + ".host", "localhost");
const auto port = static_cast<UInt16>(config.getUInt(settings_config_prefix + ".port", default_port));
configuration.emplace(Configuration{
.host = host,
.user = config.getString(settings_config_prefix + ".user", "default"),
.password = config.getString(settings_config_prefix + ".password", ""),
.quota_key = config.getString(settings_config_prefix + ".quota_key", ""),
.db = config.getString(settings_config_prefix + ".db", default_database),
.table = config.getString(settings_config_prefix + ".table", ""),
.query = config.getString(settings_config_prefix + ".query", ""),
.where = config.getString(settings_config_prefix + ".where", ""),
.invalidate_query = config.getString(settings_config_prefix + ".invalidate_query", ""),
.update_field = config.getString(settings_config_prefix + ".update_field", ""),
.update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1),
.port = port,
.is_local = isLocalAddress({host, port}, default_port),
.secure = secure,
});
}
ClickHouseDictionarySource::Configuration configuration{
.host = host,
.user = user,
.password = password,
.quota_key = quota_key,
.db = db,
.table = table,
.query = config.getString(settings_config_prefix + ".query", ""),
.where = config.getString(settings_config_prefix + ".where", ""),
.invalidate_query = config.getString(settings_config_prefix + ".invalidate_query", ""),
.update_field = config.getString(settings_config_prefix + ".update_field", ""),
.update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1),
.port = port,
.is_local = isLocalAddress({host, port}, default_port),
.secure = config.getBool(settings_config_prefix + ".secure", false)};
ContextMutablePtr context;
if (configuration.is_local)
if (configuration->is_local)
{
/// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication).
Session session(global_context, ClientInfo::Interface::LOCAL);
session.authenticate(configuration.user, configuration.password, Poco::Net::SocketAddress{});
session.authenticate(configuration->user, configuration->password, Poco::Net::SocketAddress{});
context = session.makeQueryContext();
}
else
@ -277,7 +286,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory)
context = Context::createCopy(global_context);
if (created_from_ddl)
context->getRemoteHostFilter().checkHostAndPort(configuration.host, toString(configuration.port));
context->getRemoteHostFilter().checkHostAndPort(configuration->host, toString(configuration->port));
}
context->applySettingsChanges(readSettingsFromDictionaryConfig(config, config_prefix));
@ -285,10 +294,10 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory)
String dictionary_name = config.getString(".dictionary.name", "");
String dictionary_database = config.getString(".dictionary.database", "");
if (dictionary_name == configuration.table && dictionary_database == configuration.db)
if (dictionary_name == configuration->table && dictionary_database == configuration->db)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "ClickHouseDictionarySource table cannot be dictionary table");
return std::make_unique<ClickHouseDictionarySource>(dict_struct, configuration, sample_block, context);
return std::make_unique<ClickHouseDictionarySource>(dict_struct, *configuration, sample_block, context);
};
factory.registerSource("clickhouse", create_table_source);

2
src/Disks/TemporaryFileOnDisk.cpp
View File

@ -27,7 +27,7 @@ TemporaryFileOnDisk::TemporaryFileOnDisk(const DiskPtr & disk_)
: TemporaryFileOnDisk(disk_, "")
{}
TemporaryFileOnDisk::TemporaryFileOnDisk(const DiskPtr & disk_, CurrentMetrics::Value metric_scope)
TemporaryFileOnDisk::TemporaryFileOnDisk(const DiskPtr & disk_, CurrentMetrics::Metric metric_scope)
: TemporaryFileOnDisk(disk_)
{
sub_metric_increment.emplace(metric_scope);

2
src/Disks/TemporaryFileOnDisk.h
View File

@ -17,7 +17,7 @@ class TemporaryFileOnDisk
{
public:
explicit TemporaryFileOnDisk(const DiskPtr & disk_);
explicit TemporaryFileOnDisk(const DiskPtr & disk_, CurrentMetrics::Value metric_scope);
explicit TemporaryFileOnDisk(const DiskPtr & disk_, CurrentMetrics::Metric metric_scope);
explicit TemporaryFileOnDisk(const DiskPtr & disk_, const String & prefix);
~TemporaryFileOnDisk();

6
src/Functions/array/arrayAggregation.cpp
View File

@ -223,7 +223,8 @@ struct ArrayAggregateImpl
auto result_scale = column_typed->getScale() * array_size;
if (unlikely(result_scale > DecimalUtils::max_precision<AggregationType>))
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds", result_scale);
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds (max scale: {})",
result_scale, DecimalUtils::max_precision<AggregationType>);
res[i] = DecimalUtils::convertTo<ResultType>(product, static_cast<UInt32>(result_scale));
}
@ -332,7 +333,8 @@ struct ArrayAggregateImpl
auto result_scale = column->getScale() * count;
if (unlikely(result_scale > DecimalUtils::max_precision<AggregationType>))
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds", result_scale);
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds (max scale: {})",
result_scale, DecimalUtils::max_precision<AggregationType>);
res[i] = DecimalUtils::convertTo<ResultType>(aggregate_value, static_cast<UInt32>(result_scale));
}

42
src/Functions/parseDateTime.cpp
View File

@ -20,9 +20,7 @@ namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NOT_IMPLEMENTED;
extern const int BAD_ARGUMENTS;
extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE;
@ -480,33 +478,15 @@ namespace
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
if (arguments.size() != 1 && arguments.size() != 2 && arguments.size() != 3)
throw Exception(
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Number of arguments for function {} doesn't match: passed {}, should be 1, 2 or 3",
getName(),
arguments.size());
FunctionArgumentDescriptors args{
{"time", &isString<IDataType>, nullptr, "String"},
{"format", &isString<IDataType>, nullptr, "String"},
};
if (!isString(arguments[0].type))
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of first argument of function {}. Should be String",
arguments[0].type->getName(),
getName());
if (arguments.size() == 3)
args.emplace_back(FunctionArgumentDescriptor{"timezone", &isString<IDataType>, nullptr, "String"});
if (arguments.size() > 1 && !isString(arguments[1].type))
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of second argument of function {}. Should be String",
arguments[0].type->getName(),
getName());
if (arguments.size() > 2 && !isString(arguments[2].type))
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of third argument of function {}. Should be String",
arguments[0].type->getName(),
getName());
validateFunctionArgumentTypes(*this, arguments, args);
String time_zone_name = getTimeZone(arguments).getTimeZone();
DataTypePtr date_type = std::make_shared<DataTypeDateTime>(time_zone_name);
@ -1776,14 +1756,6 @@ namespace
String getFormat(const ColumnsWithTypeAndName & arguments) const
{
if (arguments.size() < 2)
{
if constexpr (parse_syntax == ParseSyntax::Joda)
return "yyyy-MM-dd HH:mm:ss";
else
return "%Y-%m-%d %H:%M:%S";
}
const auto * format_column = checkAndGetColumnConst<ColumnString>(arguments[1].column.get());
if (!format_column)
throw Exception(

5
src/Functions/pointInEllipses.cpp
View File

@ -71,8 +71,9 @@ private:
/// For array on stack, see below.
if (arguments.size() > 10000)
{
throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, "Number of arguments of function {} is too large.",
getName());
throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION,
"Number of arguments of function {} is too large (maximum: 10000).",
getName());
}
for (const auto arg_idx : collections::range(0, arguments.size()))

4
src/Functions/pointInPolygon.cpp
View File

@ -37,7 +37,7 @@ namespace DB
{
namespace ErrorCodes
{
extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int BAD_ARGUMENTS;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int ILLEGAL_COLUMN;
@ -87,7 +87,7 @@ public:
{
if (arguments.size() < 2)
{
throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Too few arguments");
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires at least 2 arguments", getName());
}
/** We allow function invocation in one of the following forms:

11
src/Functions/svg.cpp
View File

@ -13,8 +13,7 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
class FunctionSvg : public IFunction
@ -48,13 +47,9 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (arguments.size() > 2)
if (arguments.empty() || arguments.size() > 2)
{
throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, "Too many arguments");
}
else if (arguments.empty())
{
throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Too few arguments");
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Incorrect number of arguments: expected 1 or 2 arguments");
}
else if (arguments.size() == 2 && checkAndGetDataType<DataTypeString>(arguments[1].get()) == nullptr)
{

3
src/IO/ReadHelpers.h
View File

@ -165,7 +165,8 @@ void readVectorBinary(std::vector<T> & v, ReadBuffer & buf)
readVarUInt(size, buf);
if (size > DEFAULT_MAX_STRING_SIZE)
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size.");
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"Too large array size (maximum: {})", DEFAULT_MAX_STRING_SIZE);
v.resize(size);
for (size_t i = 0; i < size; ++i)

2
src/Interpreters/ActionsDAG.cpp
View File

@ -1051,7 +1051,7 @@ ActionsDAGPtr ActionsDAG::clone() const
void ActionsDAG::compileExpressions(size_t min_count_to_compile_expression, const std::unordered_set<const ActionsDAG::Node *> & lazy_executed_nodes)
{
compileFunctions(min_count_to_compile_expression, lazy_executed_nodes);
removeUnusedActions();
removeUnusedActions(/*allow_remove_inputs = */ false);
}
#endif

42
src/Interpreters/AddIndexConstraintsOptimizer.cpp
View File

@ -41,35 +41,35 @@ namespace
return true;
}
const std::unordered_map<std::string, ComparisonGraph::CompareResult> & getRelationMap()
const std::unordered_map<std::string, ComparisonGraphCompareResult> & getRelationMap()
{
const static std::unordered_map<std::string, ComparisonGraph::CompareResult> relations =
const static std::unordered_map<std::string, ComparisonGraphCompareResult> relations =
{
{"equals", ComparisonGraph::CompareResult::EQUAL},
{"less", ComparisonGraph::CompareResult::LESS},
{"lessOrEquals", ComparisonGraph::CompareResult::LESS_OR_EQUAL},
{"greaterOrEquals", ComparisonGraph::CompareResult::GREATER_OR_EQUAL},
{"greater", ComparisonGraph::CompareResult::GREATER},
{"equals", ComparisonGraphCompareResult::EQUAL},
{"less", ComparisonGraphCompareResult::LESS},
{"lessOrEquals", ComparisonGraphCompareResult::LESS_OR_EQUAL},
{"greaterOrEquals", ComparisonGraphCompareResult::GREATER_OR_EQUAL},
{"greater", ComparisonGraphCompareResult::GREATER},
};
return relations;
}
const std::unordered_map<ComparisonGraph::CompareResult, std::string> & getReverseRelationMap()
const std::unordered_map<ComparisonGraphCompareResult, std::string> & getReverseRelationMap()
{
const static std::unordered_map<ComparisonGraph::CompareResult, std::string> relations =
const static std::unordered_map<ComparisonGraphCompareResult, std::string> relations =
{
{ComparisonGraph::CompareResult::EQUAL, "equals"},
{ComparisonGraph::CompareResult::LESS, "less"},
{ComparisonGraph::CompareResult::LESS_OR_EQUAL, "lessOrEquals"},
{ComparisonGraph::CompareResult::GREATER_OR_EQUAL, "greaterOrEquals"},
{ComparisonGraph::CompareResult::GREATER, "greater"},
{ComparisonGraphCompareResult::EQUAL, "equals"},
{ComparisonGraphCompareResult::LESS, "less"},
{ComparisonGraphCompareResult::LESS_OR_EQUAL, "lessOrEquals"},
{ComparisonGraphCompareResult::GREATER_OR_EQUAL, "greaterOrEquals"},
{ComparisonGraphCompareResult::GREATER, "greater"},
};
return relations;
}
bool canBeSequence(const ComparisonGraph::CompareResult left, const ComparisonGraph::CompareResult right)
bool canBeSequence(const ComparisonGraphCompareResult left, const ComparisonGraphCompareResult right)
{
using CR = ComparisonGraph::CompareResult;
using CR = ComparisonGraphCompareResult;
if (left == CR::UNKNOWN || right == CR::UNKNOWN || left == CR::NOT_EQUAL || right == CR::NOT_EQUAL)
return false;
if ((left == CR::GREATER || left == CR::GREATER_OR_EQUAL) && (right == CR::LESS || right == CR::LESS_OR_EQUAL))
@ -79,9 +79,9 @@ namespace
return true;
}
ComparisonGraph::CompareResult mostStrict(const ComparisonGraph::CompareResult left, const ComparisonGraph::CompareResult right)
ComparisonGraphCompareResult mostStrict(const ComparisonGraphCompareResult left, const ComparisonGraphCompareResult right)
{
using CR = ComparisonGraph::CompareResult;
using CR = ComparisonGraphCompareResult;
if (left == CR::LESS || left == CR::GREATER)
return left;
if (right == CR::LESS || right == CR::GREATER)
@ -104,7 +104,7 @@ namespace
/// we can add to expression 'indexHint(I < A)' condition.
CNFQuery::OrGroup createIndexHintGroup(
const CNFQuery::OrGroup & group,
const ComparisonGraph & graph,
const ComparisonGraph<ASTPtr> & graph,
const ASTs & primary_key_only_asts)
{
CNFQuery::OrGroup result;
@ -113,14 +113,14 @@ namespace
const auto * func = atom.ast->as<ASTFunction>();
if (func && func->arguments->children.size() == 2 && getRelationMap().contains(func->name))
{
auto check_and_insert = [&](const size_t index, const ComparisonGraph::CompareResult need_result)
auto check_and_insert = [&](const size_t index, const ComparisonGraphCompareResult need_result)
{
if (!onlyConstants(func->arguments->children[1 - index]))
return false;
for (const auto & primary_key_ast : primary_key_only_asts)
{
ComparisonGraph::CompareResult actual_result;
ComparisonGraphCompareResult actual_result;
if (index == 0)
actual_result = graph.compare(primary_key_ast, func->arguments->children[index]);
else

546
src/Interpreters/ComparisonGraph.cpp
View File

@ -1,10 +1,17 @@
#include <Interpreters/ComparisonGraph.h>
#include <Parsers/IAST.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/queryToString.h>
#include <Common/FieldVisitorsAccurateComparison.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/ConstantNode.h>
#include <Functions/FunctionFactory.h>
namespace DB
{
@ -17,7 +24,7 @@ namespace
{
/// Make function a > b or a >= b
ASTPtr normalizeAtom(const ASTPtr & atom)
ASTPtr normalizeAtom(const ASTPtr & atom, ContextPtr)
{
static const std::map<std::string, std::string> inverse_relations =
{
@ -29,26 +36,158 @@ ASTPtr normalizeAtom(const ASTPtr & atom)
if (const auto * func = res->as<ASTFunction>())
{
if (const auto it = inverse_relations.find(func->name); it != std::end(inverse_relations))
{
res = makeASTFunction(it->second, func->arguments->children[1]->clone(), func->arguments->children[0]->clone());
}
}
return res;
}
QueryTreeNodePtr normalizeAtom(const QueryTreeNodePtr & atom, const ContextPtr & context)
{
static const std::map<std::string, std::string> inverse_relations =
{
{"lessOrEquals", "greaterOrEquals"},
{"less", "greater"},
};
if (const auto * function_node = atom->as<FunctionNode>())
{
if (const auto it = inverse_relations.find(function_node->getFunctionName()); it != inverse_relations.end())
{
auto inverted_node = function_node->clone();
auto * inverted_function_node = inverted_node->as<FunctionNode>();
auto function_resolver = FunctionFactory::instance().get(it->second, context);
auto & arguments = inverted_function_node->getArguments().getNodes();
assert(arguments.size() == 2);
std::swap(arguments[0], arguments[1]);
inverted_function_node->resolveAsFunction(function_resolver);
return inverted_node;
}
}
return atom;
}
const FunctionNode * tryGetFunctionNode(const QueryTreeNodePtr & node)
{
return node->as<FunctionNode>();
}
const ASTFunction * tryGetFunctionNode(const ASTPtr & node)
{
return node->as<ASTFunction>();
}
std::string functionName(const QueryTreeNodePtr & node)
{
return node->as<FunctionNode &>().getFunctionName();
}
std::string functionName(const ASTPtr & node)
{
return node->as<ASTFunction &>().name;
}
const Field * tryGetConstantValue(const QueryTreeNodePtr & node)
{
if (const auto * constant = node->as<ConstantNode>())
return &constant->getValue();
return nullptr;
}
const Field * tryGetConstantValue(const ASTPtr & node)
{
if (const auto * constant = node->as<ASTLiteral>())
return &constant->value;
return nullptr;
}
template <typename Node>
const Field & getConstantValue(const Node & node)
{
const auto * constant = tryGetConstantValue(node);
assert(constant);
return *constant;
}
const auto & getNode(const Analyzer::CNF::AtomicFormula & atom)
{
return atom.node_with_hash.node;
}
const auto & getNode(const CNFQuery::AtomicFormula & atom)
{
return atom.ast;
}
std::string nodeToString(const ASTPtr & ast)
{
return queryToString(ast);
}
std::string nodeToString(const QueryTreeNodePtr & node)
{
return queryToString(node->toAST());
}
const auto & getArguments(const ASTFunction * function)
{
return function->arguments->children;
}
const auto & getArguments(const FunctionNode * function)
{
return function->getArguments().getNodes();
}
bool less(const Field & lhs, const Field & rhs) { return applyVisitor(FieldVisitorAccurateLess{}, lhs, rhs); }
bool greater(const Field & lhs, const Field & rhs) { return applyVisitor(FieldVisitorAccurateLess{}, rhs, lhs); }
bool equals(const Field & lhs, const Field & rhs) { return applyVisitor(FieldVisitorAccurateEquals{}, lhs, rhs); }
ComparisonGraphCompareResult functionNameToCompareResult(const std::string & name)
{
using enum ComparisonGraphCompareResult;
static const std::unordered_map<std::string, ComparisonGraphCompareResult> relation_to_compare =
{
{"equals", EQUAL},
{"notEquals", NOT_EQUAL},
{"less", LESS},
{"lessOrEquals", LESS_OR_EQUAL},
{"greaterOrEquals", GREATER_OR_EQUAL},
{"greater", GREATER},
};
const auto it = relation_to_compare.find(name);
return it == std::end(relation_to_compare) ? UNKNOWN : it->second;
}
ComparisonGraph::ComparisonGraph(const ASTs & atomic_formulas)
ComparisonGraphCompareResult inverseCompareResult(ComparisonGraphCompareResult result)
{
using enum ComparisonGraphCompareResult;
static const std::unordered_map<ComparisonGraphCompareResult, ComparisonGraphCompareResult> inverse_relations =
{
{NOT_EQUAL, EQUAL},
{EQUAL, NOT_EQUAL},
{GREATER_OR_EQUAL, LESS},
{GREATER, LESS_OR_EQUAL},
{LESS, GREATER_OR_EQUAL},
{LESS_OR_EQUAL, GREATER},
{UNKNOWN, UNKNOWN},
};
return inverse_relations.at(result);
}
}
template <ComparisonGraphNodeType Node>
ComparisonGraph<Node>::ComparisonGraph(const NodeContainer & atomic_formulas, ContextPtr context)
{
if (atomic_formulas.empty())
return;
static const std::unordered_map<std::string, Edge::Type> relation_to_enum =
static const std::unordered_map<std::string, typename Edge::Type> relation_to_enum =
{
{"equals", Edge::EQUAL},
{"greater", Edge::GREATER},
@ -63,20 +202,23 @@ ComparisonGraph::ComparisonGraph(const ASTs & atomic_formulas)
Graph g;
for (const auto & atom_raw : atomic_formulas)
{
const auto atom = normalizeAtom(atom_raw);
const auto atom = normalizeAtom(atom_raw, context);
auto get_index = [](const ASTPtr & ast, Graph & asts_graph) -> std::optional<size_t>
auto get_index = [](const Node & node, Graph & nodes_graph) -> std::optional<size_t>
{
const auto it = asts_graph.ast_hash_to_component.find(ast->getTreeHash());
if (it != std::end(asts_graph.ast_hash_to_component))
const auto it = nodes_graph.node_hash_to_component.find(Graph::getHash(node));
if (it != std::end(nodes_graph.node_hash_to_component))
{
if (!std::any_of(
std::cbegin(asts_graph.vertices[it->second].asts),
std::cend(asts_graph.vertices[it->second].asts),
[ast](const ASTPtr & constraint_ast)
std::cbegin(nodes_graph.vertices[it->second].nodes),
std::cend(nodes_graph.vertices[it->second].nodes),
[node](const Node & constraint_node)
{
return constraint_ast->getTreeHash() == ast->getTreeHash()
&& constraint_ast->getColumnName() == ast->getColumnName();
if constexpr (with_ast)
return constraint_node->getTreeHash() == node->getTreeHash()
&& constraint_node->getColumnName() == node->getColumnName();
else
return constraint_node->isEqual(*node);
}))
{
return {};
@ -86,26 +228,30 @@ ComparisonGraph::ComparisonGraph(const ASTs & atomic_formulas)
}
else
{
asts_graph.ast_hash_to_component[ast->getTreeHash()] = asts_graph.vertices.size();
asts_graph.vertices.push_back(EqualComponent{{ast}, std::nullopt});
asts_graph.edges.emplace_back();
return asts_graph.vertices.size() - 1;
nodes_graph.node_hash_to_component[Graph::getHash(node)] = nodes_graph.vertices.size();
nodes_graph.vertices.push_back(EqualComponent{{node}, std::nullopt});
nodes_graph.edges.emplace_back();
return nodes_graph.vertices.size() - 1;
}
};
const auto * func = atom->as<ASTFunction>();
if (func && func->arguments->children.size() == 2)
const auto * function_node = tryGetFunctionNode(atom);
if (function_node)
{
auto index_left = get_index(func->arguments->children[0], g);
auto index_right = get_index(func->arguments->children[1], g);
if (index_left && index_right)
const auto & arguments = getArguments(function_node);
if (arguments.size() == 2)
{
if (const auto it = relation_to_enum.find(func->name); it != std::end(relation_to_enum))
auto index_left = get_index(arguments[0], g);
auto index_right = get_index(arguments[1], g);
if (index_left && index_right)
{
g.edges[*index_left].push_back(Edge{it->second, *index_right});
if (it->second == Edge::EQUAL)
g.edges[*index_right].push_back(Edge{it->second, *index_left});
if (const auto it = relation_to_enum.find(functionName(atom)); it != std::end(relation_to_enum))
{
g.edges[*index_left].push_back(Edge{it->second, *index_right});
if (it->second == Edge::EQUAL)
g.edges[*index_right].push_back(Edge{it->second, *index_left});
}
}
}
}
@ -119,9 +265,9 @@ ComparisonGraph::ComparisonGraph(const ASTs & atomic_formulas)
/// All expressions from one equivalence class will be stored
/// in the corresponding vertex of new graph.
graph = buildGraphFromAstsGraph(g);
graph = buildGraphFromNodesGraph(g);
dists = buildDistsFromGraph(graph);
std::tie(ast_const_lower_bound, ast_const_upper_bound) = buildConstBounds();
std::tie(node_const_lower_bound, node_const_upper_bound) = buildConstBounds();
/// Find expressions that are known to be unequal.
static const std::unordered_set<String> not_equals_functions = {"notEquals", "greater"};
@ -130,36 +276,44 @@ ComparisonGraph::ComparisonGraph(const ASTs & atomic_formulas)
/// TODO: Build a graph for unequal components.
for (const auto & atom_raw : atomic_formulas)
{
const auto atom = normalizeAtom(atom_raw);
const auto * func = atom->as<ASTFunction>();
const auto atom = normalizeAtom(atom_raw, context);
if (func && not_equals_functions.contains(func->name))
const auto * function_node = tryGetFunctionNode(atom);
if (function_node && not_equals_functions.contains(functionName(atom)))
{
auto index_left = graph.ast_hash_to_component.at(func->arguments->children[0]->getTreeHash());
auto index_right = graph.ast_hash_to_component.at(func->arguments->children[1]->getTreeHash());
const auto & arguments = getArguments(function_node);
if (arguments.size() == 2)
{
auto index_left = graph.node_hash_to_component.at(Graph::getHash(arguments[0]));
auto index_right = graph.node_hash_to_component.at(Graph::getHash(arguments[1]));
if (index_left == index_right)
throw Exception(ErrorCodes::VIOLATED_CONSTRAINT,
"Found expression '{}', but its arguments considered equal according to constraints",
queryToString(atom));
if (index_left == index_right)
{
throw Exception(ErrorCodes::VIOLATED_CONSTRAINT,
"Found expression '{}', but its arguments considered equal according to constraints",
nodeToString(atom));
}
not_equal.emplace(index_left, index_right);
not_equal.emplace(index_right, index_left);
not_equal.emplace(index_left, index_right);
not_equal.emplace(index_right, index_left);
}
}
}
}
ComparisonGraph::CompareResult ComparisonGraph::pathToCompareResult(Path path, bool inverse)
template <ComparisonGraphNodeType Node>
ComparisonGraphCompareResult ComparisonGraph<Node>::pathToCompareResult(Path path, bool inverse)
{
switch (path)
{
case Path::GREATER: return inverse ? CompareResult::LESS : CompareResult::GREATER;
case Path::GREATER_OR_EQUAL: return inverse ? CompareResult::LESS_OR_EQUAL : CompareResult::GREATER_OR_EQUAL;
case Path::GREATER: return inverse ? ComparisonGraphCompareResult::LESS : ComparisonGraphCompareResult::GREATER;
case Path::GREATER_OR_EQUAL: return inverse ? ComparisonGraphCompareResult::LESS_OR_EQUAL : ComparisonGraphCompareResult::GREATER_OR_EQUAL;
}
UNREACHABLE();
}
std::optional<ComparisonGraph::Path> ComparisonGraph::findPath(size_t start, size_t finish) const
template <ComparisonGraphNodeType Node>
std::optional<typename ComparisonGraph<Node>::Path> ComparisonGraph<Node>::findPath(size_t start, size_t finish) const
{
const auto it = dists.find(std::make_pair(start, finish));
if (it == std::end(dists))
@ -170,18 +324,19 @@ std::optional<ComparisonGraph::Path> ComparisonGraph::findPath(size_t start, siz
return not_equal.contains({start, finish}) ? Path::GREATER : it->second;
}
ComparisonGraph::CompareResult ComparisonGraph::compare(const ASTPtr & left, const ASTPtr & right) const
template <ComparisonGraphNodeType Node>
ComparisonGraphCompareResult ComparisonGraph<Node>::compare(const Node & left, const Node & right) const
{
size_t start = 0;
size_t finish = 0;
/// TODO: check full ast
const auto it_left = graph.ast_hash_to_component.find(left->getTreeHash());
const auto it_right = graph.ast_hash_to_component.find(right->getTreeHash());
const auto it_left = graph.node_hash_to_component.find(Graph::getHash(left));
const auto it_right = graph.node_hash_to_component.find(Graph::getHash(right));
if (it_left == std::end(graph.ast_hash_to_component) || it_right == std::end(graph.ast_hash_to_component))
if (it_left == std::end(graph.node_hash_to_component) || it_right == std::end(graph.node_hash_to_component))
{
CompareResult result = CompareResult::UNKNOWN;
auto result = ComparisonGraphCompareResult::UNKNOWN;
{
const auto left_bound = getConstLowerBound(left);
const auto right_bound = getConstUpperBound(right);
@ -189,10 +344,10 @@ ComparisonGraph::CompareResult ComparisonGraph::compare(const ASTPtr & left, con
if (left_bound && right_bound)
{
if (greater(left_bound->first, right_bound->first))
result = CompareResult::GREATER;
result = ComparisonGraphCompareResult::GREATER;
else if (equals(left_bound->first, right_bound->first))
result = left_bound->second || right_bound->second
? CompareResult::GREATER : CompareResult::GREATER_OR_EQUAL;
? ComparisonGraphCompareResult::GREATER : ComparisonGraphCompareResult::GREATER_OR_EQUAL;
}
}
{
@ -202,10 +357,10 @@ ComparisonGraph::CompareResult ComparisonGraph::compare(const ASTPtr & left, con
if (left_bound && right_bound)
{
if (less(left_bound->first, right_bound->first))
result = CompareResult::LESS;
result = ComparisonGraphCompareResult::LESS;
else if (equals(left_bound->first, right_bound->first))
result = left_bound->second || right_bound->second
? CompareResult::LESS : CompareResult::LESS_OR_EQUAL;
? ComparisonGraphCompareResult::LESS : ComparisonGraphCompareResult::LESS_OR_EQUAL;
}
}
@ -218,7 +373,7 @@ ComparisonGraph::CompareResult ComparisonGraph::compare(const ASTPtr & left, con
}
if (start == finish)
return CompareResult::EQUAL;
return ComparisonGraphCompareResult::EQUAL;
if (auto path = findPath(start, finish))
return pathToCompareResult(*path, /*inverse=*/ false);
@ -227,93 +382,102 @@ ComparisonGraph::CompareResult ComparisonGraph::compare(const ASTPtr & left, con
return pathToCompareResult(*path, /*inverse=*/ true);
if (not_equal.contains({start, finish}))
return CompareResult::NOT_EQUAL;
return ComparisonGraphCompareResult::NOT_EQUAL;
return CompareResult::UNKNOWN;
return ComparisonGraphCompareResult::UNKNOWN;
}
bool ComparisonGraph::isPossibleCompare(CompareResult expected, const ASTPtr & left, const ASTPtr & right) const
template <ComparisonGraphNodeType Node>
bool ComparisonGraph<Node>::isPossibleCompare(ComparisonGraphCompareResult expected, const Node & left, const Node & right) const
{
const auto result = compare(left, right);
if (expected == CompareResult::UNKNOWN || result == CompareResult::UNKNOWN)
using enum ComparisonGraphCompareResult;
if (expected == UNKNOWN || result == UNKNOWN)
return true;
if (expected == result)
return true;
static const std::set<std::pair<CompareResult, CompareResult>> possible_pairs =
static const std::set<std::pair<ComparisonGraphCompareResult, ComparisonGraphCompareResult>> possible_pairs =
{
{CompareResult::EQUAL, CompareResult::LESS_OR_EQUAL},
{CompareResult::EQUAL, CompareResult::GREATER_OR_EQUAL},
{CompareResult::LESS_OR_EQUAL, CompareResult::LESS},
{CompareResult::LESS_OR_EQUAL, CompareResult::EQUAL},
{CompareResult::LESS_OR_EQUAL, CompareResult::NOT_EQUAL},
{CompareResult::GREATER_OR_EQUAL, CompareResult::GREATER},
{CompareResult::GREATER_OR_EQUAL, CompareResult::EQUAL},
{CompareResult::GREATER_OR_EQUAL, CompareResult::NOT_EQUAL},
{CompareResult::LESS, CompareResult::LESS},
{CompareResult::LESS, CompareResult::LESS_OR_EQUAL},
{CompareResult::LESS, CompareResult::NOT_EQUAL},
{CompareResult::GREATER, CompareResult::GREATER},
{CompareResult::GREATER, CompareResult::GREATER_OR_EQUAL},
{CompareResult::GREATER, CompareResult::NOT_EQUAL},
{CompareResult::NOT_EQUAL, CompareResult::LESS},
{CompareResult::NOT_EQUAL, CompareResult::GREATER},
{CompareResult::NOT_EQUAL, CompareResult::LESS_OR_EQUAL},
{CompareResult::NOT_EQUAL, CompareResult::GREATER_OR_EQUAL},
{EQUAL, LESS_OR_EQUAL},
{EQUAL, GREATER_OR_EQUAL},
{LESS_OR_EQUAL, LESS},
{LESS_OR_EQUAL, EQUAL},
{LESS_OR_EQUAL, NOT_EQUAL},
{GREATER_OR_EQUAL, GREATER},
{GREATER_OR_EQUAL, EQUAL},
{GREATER_OR_EQUAL, NOT_EQUAL},
{LESS, LESS},
{LESS, LESS_OR_EQUAL},
{LESS, NOT_EQUAL},
{GREATER, GREATER},
{GREATER, GREATER_OR_EQUAL},
{GREATER, NOT_EQUAL},
{NOT_EQUAL, LESS},
{NOT_EQUAL, GREATER},
{NOT_EQUAL, LESS_OR_EQUAL},
{NOT_EQUAL, GREATER_OR_EQUAL},
};
return possible_pairs.contains({expected, result});
}
bool ComparisonGraph::isAlwaysCompare(CompareResult expected, const ASTPtr & left, const ASTPtr & right) const
template <ComparisonGraphNodeType Node>
bool ComparisonGraph<Node>::isAlwaysCompare(ComparisonGraphCompareResult expected, const Node & left, const Node & right) const
{
const auto result = compare(left, right);
if (expected == CompareResult::UNKNOWN || result == CompareResult::UNKNOWN)
using enum ComparisonGraphCompareResult;
if (expected == UNKNOWN || result == UNKNOWN)
return false;
if (expected == result)
return true;
static const std::set<std::pair<CompareResult, CompareResult>> possible_pairs =
static const std::set<std::pair<ComparisonGraphCompareResult, ComparisonGraphCompareResult>> possible_pairs =
{
{CompareResult::LESS_OR_EQUAL, CompareResult::LESS},
{CompareResult::LESS_OR_EQUAL, CompareResult::EQUAL},
{CompareResult::GREATER_OR_EQUAL, CompareResult::GREATER},
{CompareResult::GREATER_OR_EQUAL, CompareResult::EQUAL},
{CompareResult::NOT_EQUAL, CompareResult::GREATER},
{CompareResult::NOT_EQUAL, CompareResult::LESS},
{LESS_OR_EQUAL, LESS},
{LESS_OR_EQUAL, EQUAL},
{GREATER_OR_EQUAL, GREATER},
{GREATER_OR_EQUAL, EQUAL},
{NOT_EQUAL, GREATER},
{NOT_EQUAL, LESS},
};
return possible_pairs.contains({expected, result});
}
ASTs ComparisonGraph::getEqual(const ASTPtr & ast) const
template <ComparisonGraphNodeType Node>
typename ComparisonGraph<Node>::NodeContainer ComparisonGraph<Node>::getEqual(const Node & node) const
{
const auto res = getComponentId(ast);
const auto res = getComponentId(node);
if (!res)
return {};
else
return getComponent(res.value());
}
std::optional<size_t> ComparisonGraph::getComponentId(const ASTPtr & ast) const
template <ComparisonGraphNodeType Node>
std::optional<size_t> ComparisonGraph<Node>::getComponentId(const Node & node) const
{
const auto hash_it = graph.ast_hash_to_component.find(ast->getTreeHash());
if (hash_it == std::end(graph.ast_hash_to_component))
const auto hash_it = graph.node_hash_to_component.find(Graph::getHash(node));
if (hash_it == std::end(graph.node_hash_to_component))
return {};
const size_t index = hash_it->second;
if (std::any_of(
std::cbegin(graph.vertices[index].asts),
std::cend(graph.vertices[index].asts),
[ast](const ASTPtr & constraint_ast)
std::cbegin(graph.vertices[index].nodes),
std::cend(graph.vertices[index].nodes),
[node](const Node & constraint_node)
{
return constraint_ast->getTreeHash() == ast->getTreeHash() &&
constraint_ast->getColumnName() == ast->getColumnName();
if constexpr (with_ast)
return constraint_node->getTreeHash() == node->getTreeHash()
&& constraint_node->getColumnName() == node->getColumnName();
else
return constraint_node->getTreeHash() == node->getTreeHash();
}))
{
return index;
@ -324,33 +488,38 @@ std::optional<size_t> ComparisonGraph::getComponentId(const ASTPtr & ast) const
}
}
bool ComparisonGraph::hasPath(size_t left, size_t right) const
template <ComparisonGraphNodeType Node>
bool ComparisonGraph<Node>::hasPath(size_t left, size_t right) const
{
return findPath(left, right) || findPath(right, left);
}
ASTs ComparisonGraph::getComponent(size_t id) const
template <ComparisonGraphNodeType Node>
typename ComparisonGraph<Node>::NodeContainer ComparisonGraph<Node>::getComponent(size_t id) const
{
return graph.vertices[id].asts;
return graph.vertices[id].nodes;
}
bool ComparisonGraph::EqualComponent::hasConstant() const
template <ComparisonGraphNodeType Node>
bool ComparisonGraph<Node>::EqualComponent::hasConstant() const
{
return constant_index.has_value();
}
ASTPtr ComparisonGraph::EqualComponent::getConstant() const
template <ComparisonGraphNodeType Node>
Node ComparisonGraph<Node>::EqualComponent::getConstant() const
{
assert(constant_index);
return asts[*constant_index];
return nodes[*constant_index];
}
void ComparisonGraph::EqualComponent::buildConstants()
template <ComparisonGraphNodeType Node>
void ComparisonGraph<Node>::EqualComponent::buildConstants()
{
constant_index.reset();
for (size_t i = 0; i < asts.size(); ++i)
for (size_t i = 0; i < nodes.size(); ++i)
{
if (asts[i]->as<ASTLiteral>())
if (tryGetConstantValue(nodes[i]) != nullptr)
{
constant_index = i;
return;
@ -358,133 +527,120 @@ void ComparisonGraph::EqualComponent::buildConstants()
}
}
ComparisonGraph::CompareResult ComparisonGraph::atomToCompareResult(const CNFQuery::AtomicFormula & atom)
template <ComparisonGraphNodeType Node>
ComparisonGraphCompareResult ComparisonGraph<Node>::atomToCompareResult(const typename CNF::AtomicFormula & atom)
{
if (const auto * func = atom.ast->as<ASTFunction>())
const auto & node = getNode(atom);
if (tryGetFunctionNode(node) != nullptr)
{
auto expected = functionNameToCompareResult(func->name);
auto expected = functionNameToCompareResult(functionName(node));
if (atom.negative)
expected = inverseCompareResult(expected);
return expected;
}
return ComparisonGraph::CompareResult::UNKNOWN;
return ComparisonGraphCompareResult::UNKNOWN;
}
ComparisonGraph::CompareResult ComparisonGraph::functionNameToCompareResult(const std::string & name)
template <ComparisonGraphNodeType Node>
std::optional<Node> ComparisonGraph<Node>::getEqualConst(const Node & node) const
{
static const std::unordered_map<std::string, CompareResult> relation_to_compare =
{
{"equals", CompareResult::EQUAL},
{"notEquals", CompareResult::NOT_EQUAL},
{"less", CompareResult::LESS},
{"lessOrEquals", CompareResult::LESS_OR_EQUAL},
{"greaterOrEquals", CompareResult::GREATER_OR_EQUAL},
{"greater", CompareResult::GREATER},
};
const auto it = relation_to_compare.find(name);
return it == std::end(relation_to_compare) ? CompareResult::UNKNOWN : it->second;
}
ComparisonGraph::CompareResult ComparisonGraph::inverseCompareResult(CompareResult result)
{
static const std::unordered_map<CompareResult, CompareResult> inverse_relations =
{
{CompareResult::NOT_EQUAL, CompareResult::EQUAL},
{CompareResult::EQUAL, CompareResult::NOT_EQUAL},
{CompareResult::GREATER_OR_EQUAL, CompareResult::LESS},
{CompareResult::GREATER, CompareResult::LESS_OR_EQUAL},
{CompareResult::LESS, CompareResult::GREATER_OR_EQUAL},
{CompareResult::LESS_OR_EQUAL, CompareResult::GREATER},
{CompareResult::UNKNOWN, CompareResult::UNKNOWN},
};
return inverse_relations.at(result);
}
std::optional<ASTPtr> ComparisonGraph::getEqualConst(const ASTPtr & ast) const
{
const auto hash_it = graph.ast_hash_to_component.find(ast->getTreeHash());
if (hash_it == std::end(graph.ast_hash_to_component))
const auto hash_it = graph.node_hash_to_component.find(Graph::getHash(node));
if (hash_it == std::end(graph.node_hash_to_component))
return std::nullopt;
const size_t index = hash_it->second;
return graph.vertices[index].hasConstant()
? std::optional<ASTPtr>{graph.vertices[index].getConstant()}
: std::nullopt;
if (!graph.vertices[index].hasConstant())
return std::nullopt;
if constexpr (with_ast)
return graph.vertices[index].getConstant();
else
{
const auto & constant = getConstantValue(graph.vertices[index].getConstant());
auto constant_node = std::make_shared<ConstantNode>(constant, node->getResultType());
return constant_node;
}
}
std::optional<std::pair<Field, bool>> ComparisonGraph::getConstUpperBound(const ASTPtr & ast) const
template <ComparisonGraphNodeType Node>
std::optional<std::pair<Field, bool>> ComparisonGraph<Node>::getConstUpperBound(const Node & node) const
{
if (const auto * literal = ast->as<ASTLiteral>())
return std::make_pair(literal->value, false);
if (const auto * constant = tryGetConstantValue(node))
return std::make_pair(*constant, false);
const auto it = graph.ast_hash_to_component.find(ast->getTreeHash());
if (it == std::end(graph.ast_hash_to_component))
const auto it = graph.node_hash_to_component.find(Graph::getHash(node));
if (it == std::end(graph.node_hash_to_component))
return std::nullopt;
const size_t to = it->second;
const ssize_t from = ast_const_upper_bound[to];
const ssize_t from = node_const_upper_bound[to];
if (from == -1)
return std::nullopt;
return std::make_pair(graph.vertices[from].getConstant()->as<ASTLiteral>()->value, dists.at({from, to}) == Path::GREATER);
return std::make_pair(getConstantValue(graph.vertices[from].getConstant()), dists.at({from, to}) == Path::GREATER);
}
std::optional<std::pair<Field, bool>> ComparisonGraph::getConstLowerBound(const ASTPtr & ast) const
template <ComparisonGraphNodeType Node>
std::optional<std::pair<Field, bool>> ComparisonGraph<Node>::getConstLowerBound(const Node & node) const
{
if (const auto * literal = ast->as<ASTLiteral>())
return std::make_pair(literal->value, false);
if (const auto * constant = tryGetConstantValue(node))
return std::make_pair(*constant, false);
const auto it = graph.ast_hash_to_component.find(ast->getTreeHash());
if (it == std::end(graph.ast_hash_to_component))
const auto it = graph.node_hash_to_component.find(Graph::getHash(node));
if (it == std::end(graph.node_hash_to_component))
return std::nullopt;
const size_t from = it->second;
const ssize_t to = ast_const_lower_bound[from];
const ssize_t to = node_const_lower_bound[from];
if (to == -1)
return std::nullopt;
return std::make_pair(graph.vertices[to].getConstant()->as<ASTLiteral>()->value, dists.at({from, to}) == Path::GREATER);
return std::make_pair(getConstantValue(graph.vertices[to].getConstant()), dists.at({from, to}) == Path::GREATER);
}
void ComparisonGraph::dfsOrder(const Graph & asts_graph, size_t v, std::vector<bool> & visited, std::vector<size_t> & order)
template <ComparisonGraphNodeType Node>
void ComparisonGraph<Node>::dfsOrder(const Graph & nodes_graph, size_t v, std::vector<bool> & visited, std::vector<size_t> & order)
{
visited[v] = true;
for (const auto & edge : asts_graph.edges[v])
for (const auto & edge : nodes_graph.edges[v])
if (!visited[edge.to])
dfsOrder(asts_graph, edge.to, visited, order);
dfsOrder(nodes_graph, edge.to, visited, order);
order.push_back(v);
}
ComparisonGraph::Graph ComparisonGraph::reverseGraph(const Graph & asts_graph)
template <ComparisonGraphNodeType Node>
typename ComparisonGraph<Node>::Graph ComparisonGraph<Node>::reverseGraph(const Graph & nodes_graph)
{
Graph g;
g.ast_hash_to_component = asts_graph.ast_hash_to_component;
g.vertices = asts_graph.vertices;
g.node_hash_to_component = nodes_graph.node_hash_to_component;
g.vertices = nodes_graph.vertices;
g.edges.resize(g.vertices.size());
for (size_t v = 0; v < asts_graph.vertices.size(); ++v)
for (const auto & edge : asts_graph.edges[v])
for (size_t v = 0; v < nodes_graph.vertices.size(); ++v)
for (const auto & edge : nodes_graph.edges[v])
g.edges[edge.to].push_back(Edge{edge.type, v});
return g;
}
std::vector<ASTs> ComparisonGraph::getVertices() const
template <ComparisonGraphNodeType Node>
std::vector<typename ComparisonGraph<Node>::NodeContainer> ComparisonGraph<Node>::getVertices() const
{
std::vector<ASTs> result;
std::vector<NodeContainer> result;
for (const auto & vertex : graph.vertices)
{
result.emplace_back();
for (const auto & ast : vertex.asts)
result.back().push_back(ast);
for (const auto & node : vertex.nodes)
result.back().push_back(node);
}
return result;
}
void ComparisonGraph::dfsComponents(
template <ComparisonGraphNodeType Node>
void ComparisonGraph<Node>::dfsComponents(
const Graph & reversed_graph, size_t v,
OptionalIndices & components, size_t component)
{
@ -494,11 +650,12 @@ void ComparisonGraph::dfsComponents(
dfsComponents(reversed_graph, edge.to, components, component);
}
ComparisonGraph::Graph ComparisonGraph::buildGraphFromAstsGraph(const Graph & asts_graph)
template <ComparisonGraphNodeType Node>
typename ComparisonGraph<Node>::Graph ComparisonGraph<Node>::buildGraphFromNodesGraph(const Graph & nodes_graph)
{
/// Find strongly connected component by using 2 dfs traversals.
/// https://en.wikipedia.org/wiki/Kosaraju%27s_algorithm
const auto n = asts_graph.vertices.size();
const auto n = nodes_graph.vertices.size();
std::vector<size_t> order;
{
@ -506,14 +663,14 @@ ComparisonGraph::Graph ComparisonGraph::buildGraphFromAstsGraph(const Graph & as
for (size_t v = 0; v < n; ++v)
{
if (!visited[v])
dfsOrder(asts_graph, v, visited, order);
dfsOrder(nodes_graph, v, visited, order);
}
}
OptionalIndices components(n);
size_t component = 0;
{
const Graph reversed_graph = reverseGraph(asts_graph);
const Graph reversed_graph = reverseGraph(nodes_graph);
for (auto it = order.rbegin(); it != order.rend(); ++it)
{
if (!components[*it])
@ -527,14 +684,14 @@ ComparisonGraph::Graph ComparisonGraph::buildGraphFromAstsGraph(const Graph & as
Graph result;
result.vertices.resize(component);
result.edges.resize(component);
for (const auto & [hash, index] : asts_graph.ast_hash_to_component)
for (const auto & [hash, index] : nodes_graph.node_hash_to_component)
{
assert(components[index]);
result.ast_hash_to_component[hash] = *components[index];
result.vertices[*components[index]].asts.insert(
std::end(result.vertices[*components[index]].asts),
std::begin(asts_graph.vertices[index].asts),
std::end(asts_graph.vertices[index].asts)); // asts_graph has only one ast per vertex
result.node_hash_to_component[hash] = *components[index];
result.vertices[*components[index]].nodes.insert(
std::end(result.vertices[*components[index]].nodes),
std::begin(nodes_graph.vertices[index].nodes),
std::end(nodes_graph.vertices[index].nodes)); // asts_graph has only one ast per vertex
}
/// Calculate constants
@ -544,7 +701,7 @@ ComparisonGraph::Graph ComparisonGraph::buildGraphFromAstsGraph(const Graph & as
/// For each edge in initial graph, we add an edge between components in condensation graph.
for (size_t v = 0; v < n; ++v)
{
for (const auto & edge : asts_graph.edges[v])
for (const auto & edge : nodes_graph.edges[v])
result.edges[*components[v]].push_back(Edge{edge.type, *components[edge.to]});
/// TODO: make edges unique (left most strict)
@ -557,11 +714,11 @@ ComparisonGraph::Graph ComparisonGraph::buildGraphFromAstsGraph(const Graph & as
{
if (v != u && result.vertices[v].hasConstant() && result.vertices[u].hasConstant())
{
const auto * left = result.vertices[v].getConstant()->as<ASTLiteral>();
const auto * right = result.vertices[u].getConstant()->as<ASTLiteral>();
const auto & left = getConstantValue(result.vertices[v].getConstant());
const auto & right = getConstantValue(result.vertices[u].getConstant());
/// Only GREATER. Equal constant fields = equal literals so it was already considered above.
if (greater(left->value, right->value))
if (greater(left, right))
result.edges[v].push_back(Edge{Edge::GREATER, u});
}
}
@ -570,7 +727,8 @@ ComparisonGraph::Graph ComparisonGraph::buildGraphFromAstsGraph(const Graph & as
return result;
}
std::map<std::pair<size_t, size_t>, ComparisonGraph::Path> ComparisonGraph::buildDistsFromGraph(const Graph & g)
template <ComparisonGraphNodeType Node>
std::map<std::pair<size_t, size_t>, typename ComparisonGraph<Node>::Path> ComparisonGraph<Node>::buildDistsFromGraph(const Graph & g)
{
/// Min path : -1 means GREATER, 0 means GREATER_OR_EQUALS.
/// We use Floyd–Warshall algorithm to find distances between all pairs of vertices.
@ -602,7 +760,8 @@ std::map<std::pair<size_t, size_t>, ComparisonGraph::Path> ComparisonGraph::buil
return path;
}
std::pair<std::vector<ssize_t>, std::vector<ssize_t>> ComparisonGraph::buildConstBounds() const
template <ComparisonGraphNodeType Node>
std::pair<std::vector<ssize_t>, std::vector<ssize_t>> ComparisonGraph<Node>::buildConstBounds() const
{
const size_t n = graph.vertices.size();
std::vector<ssize_t> lower(n, -1);
@ -610,7 +769,7 @@ std::pair<std::vector<ssize_t>, std::vector<ssize_t>> ComparisonGraph::buildCons
auto get_value = [this](const size_t vertex) -> Field
{
return graph.vertices[vertex].getConstant()->as<ASTLiteral>()->value;
return getConstantValue(graph.vertices[vertex].getConstant());
};
for (const auto & [edge, path] : dists)
@ -634,7 +793,10 @@ std::pair<std::vector<ssize_t>, std::vector<ssize_t>> ComparisonGraph::buildCons
}
}
return {lower, upper};
return {std::move(lower), std::move(upper)};
}
template class ComparisonGraph<ASTPtr>;
template class ComparisonGraph<QueryTreeNodePtr>;
}

97
src/Interpreters/ComparisonGraph.h
View File

@ -2,6 +2,12 @@
#include <Parsers/IAST_fwd.h>
#include <Interpreters/TreeCNFConverter.h>
#include <Analyzer/Passes/CNF.h>
#include <Analyzer/HashUtils.h>
#include <Analyzer/IQueryTreeNode.h>
#include <type_traits>
#include <unordered_map>
#include <map>
#include <vector>
@ -9,50 +15,56 @@
namespace DB
{
enum class ComparisonGraphCompareResult : uint8_t
{
LESS,
LESS_OR_EQUAL,
EQUAL,
GREATER_OR_EQUAL,
GREATER,
NOT_EQUAL,
UNKNOWN,
};
template <typename T>
concept ComparisonGraphNodeType = std::same_as<T, ASTPtr> || std::same_as<T, QueryTreeNodePtr>;
/*
* Graph of relations between terms in constraints.
* Allows to compare terms and get equal terms.
*/
template <ComparisonGraphNodeType Node>
class ComparisonGraph
{
public:
static constexpr bool with_ast = std::same_as<Node, ASTPtr>;
using NodeContainer = std::conditional_t<with_ast, ASTs, QueryTreeNodes>;
using CNF = std::conditional_t<with_ast, CNFQuery, Analyzer::CNF>;
/// atomic_formulas are extracted from constraints.
explicit ComparisonGraph(const ASTs & atomic_formulas);
explicit ComparisonGraph(const NodeContainer & atomic_formulas, ContextPtr context = nullptr);
enum class CompareResult
{
LESS,
LESS_OR_EQUAL,
EQUAL,
GREATER_OR_EQUAL,
GREATER,
NOT_EQUAL,
UNKNOWN,
};
static ComparisonGraphCompareResult atomToCompareResult(const typename CNF::AtomicFormula & atom);
static CompareResult atomToCompareResult(const CNFQuery::AtomicFormula & atom);
static CompareResult functionNameToCompareResult(const std::string & name);
static CompareResult inverseCompareResult(CompareResult result);
CompareResult compare(const ASTPtr & left, const ASTPtr & right) const;
ComparisonGraphCompareResult compare(const Node & left, const Node & right) const;
/// It's possible that left <expected> right
bool isPossibleCompare(CompareResult expected, const ASTPtr & left, const ASTPtr & right) const;
bool isPossibleCompare(ComparisonGraphCompareResult expected, const Node & left, const Node & right) const;
/// It's always true that left <expected> right
bool isAlwaysCompare(CompareResult expected, const ASTPtr & left, const ASTPtr & right) const;
bool isAlwaysCompare(ComparisonGraphCompareResult expected, const Node & left, const Node & right) const;
/// Returns all expressions from component to which @ast belongs if any.
ASTs getEqual(const ASTPtr & ast) const;
/// Returns all expressions from component to which @node belongs if any.
NodeContainer getEqual(const Node & node) const;
/// Returns constant expression from component to which @ast belongs if any.
std::optional<ASTPtr> getEqualConst(const ASTPtr & ast) const;
/// Returns constant expression from component to which @node belongs if any.
std::optional<Node> getEqualConst(const Node & node) const;
/// Finds component id to which @ast belongs if any.
std::optional<std::size_t> getComponentId(const ASTPtr & ast) const;
/// Finds component id to which @node belongs if any.
std::optional<std::size_t> getComponentId(const Node & node) const;
/// Returns all expressions from component.
ASTs getComponent(size_t id) const;
NodeContainer getComponent(size_t id) const;
size_t getNumOfComponents() const { return graph.vertices.size(); }
@ -61,22 +73,22 @@ public:
/// Find constants lessOrEqual and greaterOrEqual.
/// For int and double linear programming can be applied here.
/// Returns: {constant, is strict less/greater}
std::optional<std::pair<Field, bool>> getConstUpperBound(const ASTPtr & ast) const;
std::optional<std::pair<Field, bool>> getConstLowerBound(const ASTPtr & ast) const;
std::optional<std::pair<Field, bool>> getConstUpperBound(const Node & node) const;
std::optional<std::pair<Field, bool>> getConstLowerBound(const Node & node) const;
/// Returns all expression in graph.
std::vector<ASTs> getVertices() const;
std::vector<NodeContainer> getVertices() const;
private:
/// Strongly connected component
struct EqualComponent
{
/// All these expressions are considered as equal.
ASTs asts;
NodeContainer nodes;
std::optional<size_t> constant_index;
bool hasConstant() const;
ASTPtr getConstant() const;
Node getConstant() const;
void buildConstants();
};
@ -110,20 +122,29 @@ private:
}
};
std::unordered_map<IAST::Hash, size_t, ASTHash> ast_hash_to_component;
static auto getHash(const Node & node)
{
if constexpr (with_ast)
return node->getTreeHash();
else
return QueryTreeNodePtrWithHash{node};
}
using NodeHashToComponentContainer = std::conditional_t<with_ast, std::unordered_map<IAST::Hash, size_t, ASTHash>, QueryTreeNodePtrWithHashMap<size_t>>;
NodeHashToComponentContainer node_hash_to_component;
std::vector<EqualComponent> vertices;
std::vector<std::vector<Edge>> edges;
};
/// Receives graph, in which each vertex corresponds to one expression.
/// Then finds strongly connected components and builds graph on them.
static Graph buildGraphFromAstsGraph(const Graph & asts_graph);
static Graph buildGraphFromNodesGraph(const Graph & nodes_graph);
static Graph reverseGraph(const Graph & asts_graph);
static Graph reverseGraph(const Graph & nodes_graph);
/// The first part of finding strongly connected components.
/// Finds order of exit from vertices of dfs traversal of graph.
static void dfsOrder(const Graph & asts_graph, size_t v, std::vector<bool> & visited, std::vector<size_t> & order);
static void dfsOrder(const Graph & nodes_graph, size_t v, std::vector<bool> & visited, std::vector<size_t> & order);
using OptionalIndices = std::vector<std::optional<size_t>>;
@ -139,13 +160,13 @@ private:
GREATER_OR_EQUAL,
};
static CompareResult pathToCompareResult(Path path, bool inverse);
static ComparisonGraphCompareResult pathToCompareResult(Path path, bool inverse);
std::optional<Path> findPath(size_t start, size_t finish) const;
/// Calculate @dists.
static std::map<std::pair<size_t, size_t>, Path> buildDistsFromGraph(const Graph & g);
/// Calculate @ast_const_lower_bound and @ast_const_lower_bound.
/// Calculate @nodeconst_lower_bound and @node_const_lower_bound.
std::pair<std::vector<ssize_t>, std::vector<ssize_t>> buildConstBounds() const;
/// Direct acyclic graph in which each vertex corresponds
@ -165,11 +186,11 @@ private:
/// Maximal constant value for each component that
/// is lower bound for all expressions in component.
std::vector<ssize_t> ast_const_lower_bound;
std::vector<ssize_t> node_const_lower_bound;
/// Minimal constant value for each component that
/// is upper bound for all expressions in component.
std::vector<ssize_t> ast_const_upper_bound;
std::vector<ssize_t> node_const_upper_bound;
};
}

8
src/Interpreters/DDLWorker.cpp
View File

@ -1022,10 +1022,10 @@ String DDLWorker::enqueueQuery(DDLLogEntry & entry)
{
String str_buf = node_path.substr(query_path_prefix.length());
DB::ReadBufferFromString in(str_buf);
CurrentMetrics::Metric id;
readText(id, in);
id = std::max(*max_pushed_entry_metric, id);
CurrentMetrics::set(*max_pushed_entry_metric, id);
CurrentMetrics::Value pushed_entry;
readText(pushed_entry, in);
pushed_entry = std::max(CurrentMetrics::get(*max_pushed_entry_metric), pushed_entry);
CurrentMetrics::set(*max_pushed_entry_metric, pushed_entry);
}
/// We cannot create status dirs in a single transaction with previous request,

5
src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
View File

@ -18,9 +18,12 @@ namespace DB
QueryPipelineBuilder IInterpreterUnionOrSelectQuery::buildQueryPipeline()
{
QueryPlan query_plan;
return buildQueryPipeline(query_plan);
}
QueryPipelineBuilder IInterpreterUnionOrSelectQuery::buildQueryPipeline(QueryPlan & query_plan)
{
buildQueryPlan(query_plan);
return std::move(*query_plan.buildQueryPipeline(
QueryPlanOptimizationSettings::fromContext(context), BuildQueryPipelineSettings::fromContext(context)));
}

1
src/Interpreters/IInterpreterUnionOrSelectQuery.h
View File

@ -35,6 +35,7 @@ public:
virtual void buildQueryPlan(QueryPlan & query_plan) = 0;
QueryPipelineBuilder buildQueryPipeline();
QueryPipelineBuilder buildQueryPipeline(QueryPlan & query_plan);
virtual void ignoreWithTotals() = 0;

22
src/Interpreters/InterpreterDropQuery.cpp
View File

@ -355,19 +355,22 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query,
/// Flush should not be done if shouldBeEmptyOnDetach() == false,
/// since in this case getTablesIterator() may do some additional work,
/// see DatabaseMaterializedMySQL::getTablesIterator()
for (auto iterator = database->getTablesIterator(getContext()); iterator->isValid(); iterator->next())
{
iterator->table()->flush();
}
auto table_context = Context::createCopy(getContext());
table_context->setInternalQuery(true);
/// Do not hold extra shared pointers to tables
std::vector<std::pair<String, bool>> tables_to_drop;
for (auto iterator = database->getTablesIterator(table_context); iterator->isValid(); iterator->next())
{
iterator->table()->flush();
tables_to_drop.push_back({iterator->name(), iterator->table()->isDictionary()});
}
for (const auto & table : tables_to_drop)
{
query_for_table.setTable(table.first);
query_for_table.is_dictionary = table.second;
DatabasePtr db;
UUID table_to_wait = UUIDHelpers::Nil;
query_for_table.setTable(iterator->name());
query_for_table.is_dictionary = iterator->table()->isDictionary();
executeToTableImpl(table_context, query_for_table, db, table_to_wait);
uuids_to_wait.push_back(table_to_wait);
}
@ -428,7 +431,8 @@ AccessRightsElements InterpreterDropQuery::getRequiredAccessForDDLOnCluster() co
return required_access;
}
void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool sync)
void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context,
const StorageID & target_table_id, bool sync, bool ignore_sync_setting)
{
if (DatabaseCatalog::instance().tryGetTable(target_table_id, current_context))
{
@ -445,6 +449,8 @@ void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr
/// and not allowed to drop inner table explicitly. Allowing to drop inner table without explicit grant
/// looks like expected behaviour and we have tests for it.
auto drop_context = Context::createCopy(global_context);
if (ignore_sync_setting)
drop_context->setSetting("database_atomic_wait_for_drop_and_detach_synchronously", false);
drop_context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY;
if (auto txn = current_context->getZooKeeperMetadataTransaction())
{

3
src/Interpreters/InterpreterDropQuery.h
View File

@ -24,7 +24,8 @@ public:
/// Drop table or database.
BlockIO execute() override;
static void executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool sync);
static void executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context,
const StorageID & target_table_id, bool sync, bool ignore_sync_setting = false);
bool supportsTransactions() const override;

4
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -518,15 +518,13 @@ InterpreterSelectQuery::InterpreterSelectQuery(
settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context);
ASTPtr parallel_replicas_custom_filter_ast = nullptr;
if (context->getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY && !joined_tables.tablesWithColumns().empty())
if (storage && context->getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY && !joined_tables.tablesWithColumns().empty())
{
if (settings.parallel_replicas_count > 1)
{
if (auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *context))
{
LOG_TRACE(log, "Processing query on a replica using custom_key '{}'", settings.parallel_replicas_custom_key.value);
if (!storage)
throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Storage is unknown when trying to parse custom key for parallel replica");
parallel_replicas_custom_filter_ast = getCustomKeyFilterForParallelReplica(
settings.parallel_replicas_count,

1
src/Interpreters/InterpreterSelectQueryAnalyzer.h
View File

@ -72,6 +72,7 @@ public:
void setProperClientInfo(size_t replica_number, size_t count_participating_replicas);
const Planner & getPlanner() const { return planner; }
Planner & getPlanner() { return planner; }
private:
ASTPtr query;

2
src/Interpreters/InterpreterShowProcesslistQuery.cpp
View File

@ -12,7 +12,7 @@ namespace DB
BlockIO InterpreterShowProcesslistQuery::execute()
{
return executeQuery("SELECT * FROM system.processes", getContext(), true);
return executeQuery("SELECT * FROM system.processes ORDER BY elapsed DESC", getContext(), true);
}
}

6
src/Interpreters/InterpreterSystemQuery.cpp
View File

@ -604,6 +604,7 @@ void InterpreterSystemQuery::restoreReplica()
StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextMutablePtr system_context, bool need_ddl_guard)
{
LOG_TRACE(log, "Restarting replica {}", replica);
auto table_ddl_guard = need_ddl_guard
? DatabaseCatalog::instance().getDDLGuard(replica.getDatabaseName(), replica.getTableName())
: nullptr;
@ -647,6 +648,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica,
database->attachTable(system_context, replica.table_name, table, data_path);
table->startup();
LOG_TRACE(log, "Restarted replica {}", replica);
return table;
}
@ -693,11 +695,11 @@ void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context)
guard.second = catalog.getDDLGuard(guard.first.database_name, guard.first.table_name);
size_t threads = std::min(static_cast<size_t>(getNumberOfPhysicalCPUCores()), replica_names.size());
LOG_DEBUG(log, "Will restart {} replicas using {} threads", replica_names.size(), threads);
ThreadPool pool(CurrentMetrics::RestartReplicaThreads, CurrentMetrics::RestartReplicaThreadsActive, threads);
for (auto & replica : replica_names)
{
LOG_TRACE(log, "Restarting replica on {}", replica.getNameForLogs());
pool.scheduleOrThrowOnError([&]() { tryRestartReplica(replica, system_context, false); });
}
pool.wait();
@ -896,7 +898,7 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery & query)
{
LOG_TRACE(log, "Synchronizing entries in replica's queue with table's log and waiting for current last entry to be processed");
auto sync_timeout = getContext()->getSettingsRef().receive_timeout.totalMilliseconds();
if (!storage_replicated->waitForProcessingQueue(sync_timeout, query.strict_sync))
if (!storage_replicated->waitForProcessingQueue(sync_timeout, query.sync_replica_mode))
{
LOG_ERROR(log, "SYNC REPLICA {}: Timed out!", table_id.getNameForLogs());
throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "SYNC REPLICA {}: command timed out. " \

4
src/Interpreters/MetricLog.cpp
View File

@ -50,7 +50,7 @@ void MetricLogElement::appendToBlock(MutableColumns & columns) const
columns[column_idx++]->insert(profile_events[i]);
for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i)
columns[column_idx++]->insert(current_metrics[i]);
columns[column_idx++]->insert(current_metrics[i].toUnderType());
}
@ -97,7 +97,7 @@ void MetricLog::metricThreadFunction()
elem.milliseconds = timeInMilliseconds(current_time) - timeInSeconds(current_time) * 1000;
elem.profile_events.resize(ProfileEvents::end());
for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
for (ProfileEvents::Event i = ProfileEvents::Event(0), end = ProfileEvents::end(); i < end; ++i)
{
const ProfileEvents::Count new_value = ProfileEvents::global_counters[i].load(std::memory_order_relaxed);
auto & old_value = prev_profile_events[i];

4
src/Interpreters/ProfileEventsExt.cpp
View File

@ -32,7 +32,7 @@ void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column,
auto & value_column = tuple_column.getColumn(1);
size_t size = 0;
for (Event event = 0; event < Counters::num_counters; ++event)
for (Event event = Event(0); event < Counters::num_counters; ++event)
{
UInt64 value = counters[event];
@ -54,7 +54,7 @@ static void dumpProfileEvents(ProfileEventsSnapshot const & snapshot, DB::Mutabl
size_t rows = 0;
auto & name_column = columns[NAME_COLUMN_INDEX];
auto & value_column = columns[VALUE_COLUMN_INDEX];
for (Event event = 0; event < Counters::num_counters; ++event)
for (Event event = Event(0); event < Counters::num_counters; ++event)
{
Int64 value = snapshot.counters[event];

6
src/Interpreters/SubstituteColumnOptimizer.cpp
View File

@ -32,13 +32,13 @@ public:
struct Data
{
const ComparisonGraph & graph;
const ComparisonGraph<ASTPtr> & graph;
std::set<UInt64> & components;
std::unordered_map<String, String> & old_name;
std::unordered_map<String, UInt64> & component;
UInt64 & current_id;
Data(const ComparisonGraph & graph_,
Data(const ComparisonGraph<ASTPtr> & graph_,
std::set<UInt64> & components_,
std::unordered_map<String, String> & old_name_,
std::unordered_map<String, UInt64> & component_,
@ -165,7 +165,7 @@ ColumnPrice calculatePrice(
/// price of all columns on which ast depends.
/// TODO: branch-and-bound
void bruteforce(
const ComparisonGraph & graph,
const ComparisonGraph<ASTPtr> & graph,
const std::vector<UInt64> & components,
size_t current_component,
const ColumnPriceByName & column_prices,

2
src/Interpreters/TemporaryDataOnDisk.cpp
View File

@ -49,7 +49,7 @@ TemporaryDataOnDisk::TemporaryDataOnDisk(TemporaryDataOnDiskScopePtr parent_)
: TemporaryDataOnDiskScope(std::move(parent_), /* limit_ = */ 0)
{}
TemporaryDataOnDisk::TemporaryDataOnDisk(TemporaryDataOnDiskScopePtr parent_, CurrentMetrics::Value metric_scope)
TemporaryDataOnDisk::TemporaryDataOnDisk(TemporaryDataOnDiskScopePtr parent_, CurrentMetrics::Metric metric_scope)
: TemporaryDataOnDiskScope(std::move(parent_), /* limit_ = */ 0)
, current_metric_scope(metric_scope)
{}

4
src/Interpreters/TemporaryDataOnDisk.h
View File

@ -85,7 +85,7 @@ public:
explicit TemporaryDataOnDisk(TemporaryDataOnDiskScopePtr parent_);
explicit TemporaryDataOnDisk(TemporaryDataOnDiskScopePtr parent_, CurrentMetrics::Value metric_scope);
explicit TemporaryDataOnDisk(TemporaryDataOnDiskScopePtr parent_, CurrentMetrics::Metric metric_scope);
/// If max_file_size > 0, then check that there's enough space on the disk and throw an exception in case of lack of free space
TemporaryFileStream & createStream(const Block & header, size_t max_file_size = 0);
@ -102,7 +102,7 @@ private:
mutable std::mutex mutex;
std::vector<TemporaryFileStreamPtr> streams TSA_GUARDED_BY(mutex);
typename CurrentMetrics::Value current_metric_scope = CurrentMetrics::TemporaryFilesUnknown;
typename CurrentMetrics::Metric current_metric_scope = CurrentMetrics::TemporaryFilesUnknown;
};
/*

70
src/Interpreters/TreeCNFConverter.cpp
View File

@ -360,80 +360,14 @@ CNFQuery & CNFQuery::pushNotInFunctions()
return *this;
}
namespace
{
CNFQuery::AndGroup reduceOnce(const CNFQuery::AndGroup & groups)
{
CNFQuery::AndGroup result;
for (const CNFQuery::OrGroup & group : groups)
{
CNFQuery::OrGroup copy(group);
bool inserted = false;
for (const CNFQuery::AtomicFormula & atom : group)
{
copy.erase(atom);
CNFQuery::AtomicFormula negative_atom(atom);
negative_atom.negative = !atom.negative;
copy.insert(negative_atom);
if (groups.contains(copy))
{
copy.erase(negative_atom);
result.insert(copy);
inserted = true;
break;
}
copy.erase(negative_atom);
copy.insert(atom);
}
if (!inserted)
result.insert(group);
}
return result;
}
bool isSubset(const CNFQuery::OrGroup & left, const CNFQuery::OrGroup & right)
{
if (left.size() > right.size())
return false;
for (const auto & elem : left)
if (!right.contains(elem))
return false;
return true;
}
CNFQuery::AndGroup filterSubsets(const CNFQuery::AndGroup & groups)
{
CNFQuery::AndGroup result;
for (const CNFQuery::OrGroup & group : groups)
{
bool insert = true;
for (const CNFQuery::OrGroup & other_group : groups)
{
if (isSubset(other_group, group) && group != other_group)
{
insert = false;
break;
}
}
if (insert)
result.insert(group);
}
return result;
}
}
CNFQuery & CNFQuery::reduce()
{
while (true)
{
AndGroup new_statements = reduceOnce(statements);
AndGroup new_statements = reduceOnceCNFStatements(statements);
if (statements == new_statements)
{
statements = filterSubsets(statements);
statements = filterCNFSubsets(statements);
return *this;
}
else

68
src/Interpreters/TreeCNFConverter.h
View File

@ -164,4 +164,72 @@ public:
void pushNotIn(CNFQuery::AtomicFormula & atom);
template <typename TAndGroup>
TAndGroup reduceOnceCNFStatements(const TAndGroup & groups)
{
TAndGroup result;
for (const auto & group : groups)
{
using GroupType = std::decay_t<decltype(group)>;
GroupType copy(group);
bool inserted = false;
for (const auto & atom : group)
{
copy.erase(atom);
using AtomType = std::decay_t<decltype(atom)>;
AtomType negative_atom(atom);
negative_atom.negative = !atom.negative;
copy.insert(negative_atom);
if (groups.contains(copy))
{
copy.erase(negative_atom);
result.insert(copy);
inserted = true;
break;
}
copy.erase(negative_atom);
copy.insert(atom);
}
if (!inserted)
result.insert(group);
}
return result;
}
template <typename TOrGroup>
bool isCNFGroupSubset(const TOrGroup & left, const TOrGroup & right)
{
if (left.size() > right.size())
return false;
for (const auto & elem : left)
if (!right.contains(elem))
return false;
return true;
}
template <typename TAndGroup>
TAndGroup filterCNFSubsets(const TAndGroup & groups)
{
TAndGroup result;
for (const auto & group : groups)
{
bool insert = true;
for (const auto & other_group : groups)
{
if (isCNFGroupSubset(other_group, group) && group != other_group)
{
insert = false;
break;
}
}
if (insert)
result.insert(group);
}
return result;
}
}

12
src/Interpreters/WhereConstraintsOptimizer.cpp
View File

@ -74,7 +74,7 @@ bool checkIfGroupAlwaysTrueFullMatch(const CNFQuery::OrGroup & group, const Cons
return false;
}
bool checkIfGroupAlwaysTrueGraph(const CNFQuery::OrGroup & group, const ComparisonGraph & graph)
bool checkIfGroupAlwaysTrueGraph(const CNFQuery::OrGroup & group, const ComparisonGraph<ASTPtr> & graph)
{
/// We try to find at least one atom that is always true by using comparison graph.
for (const auto & atom : group)
@ -82,7 +82,7 @@ bool checkIfGroupAlwaysTrueGraph(const CNFQuery::OrGroup & group, const Comparis
const auto * func = atom.ast->as<ASTFunction>();
if (func && func->arguments->children.size() == 2)
{
const auto expected = ComparisonGraph::atomToCompareResult(atom);
const auto expected = ComparisonGraph<ASTPtr>::atomToCompareResult(atom);
if (graph.isAlwaysCompare(expected, func->arguments->children[0], func->arguments->children[1]))
return true;
}
@ -108,20 +108,20 @@ bool checkIfAtomAlwaysFalseFullMatch(const CNFQuery::AtomicFormula & atom, const
return false;
}
bool checkIfAtomAlwaysFalseGraph(const CNFQuery::AtomicFormula & atom, const ComparisonGraph & graph)
bool checkIfAtomAlwaysFalseGraph(const CNFQuery::AtomicFormula & atom, const ComparisonGraph<ASTPtr> & graph)
{
const auto * func = atom.ast->as<ASTFunction>();
if (func && func->arguments->children.size() == 2)
{
/// TODO: special support for !=
const auto expected = ComparisonGraph::atomToCompareResult(atom);
const auto expected = ComparisonGraph<ASTPtr>::atomToCompareResult(atom);
return !graph.isPossibleCompare(expected, func->arguments->children[0], func->arguments->children[1]);
}
return false;
}
void replaceToConstants(ASTPtr & term, const ComparisonGraph & graph)
void replaceToConstants(ASTPtr & term, const ComparisonGraph<ASTPtr> & graph)
{
const auto equal_constant = graph.getEqualConst(term);
if (equal_constant)
@ -135,7 +135,7 @@ void replaceToConstants(ASTPtr & term, const ComparisonGraph & graph)
}
}
CNFQuery::AtomicFormula replaceTermsToConstants(const CNFQuery::AtomicFormula & atom, const ComparisonGraph & graph)
CNFQuery::AtomicFormula replaceTermsToConstants(const CNFQuery::AtomicFormula & atom, const ComparisonGraph<ASTPtr> & graph)
{
CNFQuery::AtomicFormula result;
result.negative = atom.negative;

55
src/Interpreters/executeDDLQueryOnCluster.cpp
View File

@ -37,6 +37,18 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
static ZooKeeperRetriesInfo getRetriesInfo()
{
const auto & config_ref = Context::getGlobalContextInstance()->getConfigRef();
return ZooKeeperRetriesInfo(
"DistributedDDL",
&Poco::Logger::get("DDLQueryStatusSource"),
config_ref.getInt("distributed_ddl_keeper_max_retries", 5),
config_ref.getInt("distributed_ddl_keeper_initial_backoff_ms", 100),
config_ref.getInt("distributed_ddl_keeper_max_backoff_ms", 5000)
);
}
bool isSupportedAlterType(int type)
{
assert(type != ASTAlterCommand::NO_TYPE);
@ -174,7 +186,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context,
entry.tracing_context = OpenTelemetry::CurrentContext();
String node_path = ddl_worker.enqueueQuery(entry);
return getDistributedDDLStatus(node_path, entry, context);
return getDistributedDDLStatus(node_path, entry, context, /* hosts_to_wait */ nullptr);
}
@ -182,7 +194,7 @@ class DDLQueryStatusSource final : public ISource
{
public:
DDLQueryStatusSource(
const String & zk_node_path, const DDLLogEntry & entry, ContextPtr context_, const std::optional<Strings> & hosts_to_wait = {});
const String & zk_node_path, const DDLLogEntry & entry, ContextPtr context_, const Strings * hosts_to_wait);
String getName() const override { return "DDLQueryStatus"; }
Chunk generate() override;
@ -230,7 +242,7 @@ private:
};
BlockIO getDistributedDDLStatus(const String & node_path, const DDLLogEntry & entry, ContextPtr context, const std::optional<Strings> & hosts_to_wait)
BlockIO getDistributedDDLStatus(const String & node_path, const DDLLogEntry & entry, ContextPtr context, const Strings * hosts_to_wait)
{
BlockIO io;
if (context->getSettingsRef().distributed_ddl_task_timeout == 0)
@ -291,8 +303,8 @@ Block DDLQueryStatusSource::getSampleBlock(ContextPtr context_, bool hosts_to_wa
}
DDLQueryStatusSource::DDLQueryStatusSource(
const String & zk_node_path, const DDLLogEntry & entry, ContextPtr context_, const std::optional<Strings> & hosts_to_wait)
: ISource(getSampleBlock(context_, hosts_to_wait.has_value()))
const String & zk_node_path, const DDLLogEntry & entry, ContextPtr context_, const Strings * hosts_to_wait)
: ISource(getSampleBlock(context_, static_cast<bool>(hosts_to_wait)))
, node_path(zk_node_path)
, context(context_)
, watch(CLOCK_MONOTONIC_COARSE)
@ -380,7 +392,6 @@ Chunk DDLQueryStatusSource::generate()
if (is_replicated_database && context->getSettingsRef().database_replicated_enforce_synchronous_settings)
node_to_wait = "synced";
auto zookeeper = context->getZooKeeper();
size_t try_number = 0;
while (true)
@ -420,7 +431,23 @@ Chunk DDLQueryStatusSource::generate()
sleepForMilliseconds(std::min<size_t>(1000, 50 * (try_number + 1)));
}
if (!zookeeper->exists(node_path))
bool node_exists = false;
Strings tmp_hosts;
Strings tmp_active_hosts;
{
auto retries_info = getRetriesInfo();
auto retries_ctl = ZooKeeperRetriesControl("executeDDLQueryOnCluster", retries_info);
retries_ctl.retryLoop([&]()
{
auto zookeeper = context->getZooKeeper();
node_exists = zookeeper->exists(node_path);
tmp_hosts = getChildrenAllowNoNode(zookeeper, fs::path(node_path) / node_to_wait);
tmp_active_hosts = getChildrenAllowNoNode(zookeeper, fs::path(node_path) / "active");
});
}
if (!node_exists)
{
/// Paradoxically, this exception will be throw even in case of "never_throw" mode.
@ -432,12 +459,12 @@ Chunk DDLQueryStatusSource::generate()
return {};
}
Strings new_hosts = getNewAndUpdate(getChildrenAllowNoNode(zookeeper, fs::path(node_path) / node_to_wait));
Strings new_hosts = getNewAndUpdate(tmp_hosts);
++try_number;
if (new_hosts.empty())
continue;
current_active_hosts = getChildrenAllowNoNode(zookeeper, fs::path(node_path) / "active");
current_active_hosts = std::move(tmp_active_hosts);
MutableColumns columns = output.getHeader().cloneEmptyColumns();
for (const String & host_id : new_hosts)
@ -447,7 +474,15 @@ Chunk DDLQueryStatusSource::generate()
if (node_to_wait == "finished")
{
String status_data;
if (zookeeper->tryGet(fs::path(node_path) / "finished" / host_id, status_data))
bool finished_exists = false;
auto retries_info = getRetriesInfo();
auto retries_ctl = ZooKeeperRetriesControl("executeDDLQueryOnCluster", retries_info);
retries_ctl.retryLoop([&]()
{
finished_exists = context->getZooKeeper()->tryGet(fs::path(node_path) / "finished" / host_id, status_data);
});
if (finished_exists)
status.tryDeserializeText(status_data);
}
else

4
src/Interpreters/executeDDLQueryOnCluster.h
View File

@ -5,6 +5,7 @@
#include <Processors/ISource.h>
#include <Interpreters/Context_fwd.h>
#include <Parsers/IAST_fwd.h>
#include <Storages/MergeTree/ZooKeeperRetries.h>
namespace zkutil
@ -42,8 +43,7 @@ struct DDLQueryOnClusterParams
/// Returns DDLQueryStatusSource, which reads results of query execution on each host in the cluster.
BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, ContextPtr context, const DDLQueryOnClusterParams & params = {});
BlockIO getDistributedDDLStatus(
const String & node_path, const DDLLogEntry & entry, ContextPtr context, const std::optional<Strings> & hosts_to_wait = {});
BlockIO getDistributedDDLStatus(const String & node_path, const DDLLogEntry & entry, ContextPtr context, const Strings * hosts_to_wait);
bool maybeRemoveOnCluster(const ASTPtr & query_ptr, ContextPtr context);

48
src/Interpreters/tests/gtest_comparison_graph.cpp
View File

@ -9,11 +9,11 @@
using namespace DB;
static ComparisonGraph getGraph(const String & query)
static ComparisonGraph<ASTPtr> getGraph(const String & query)
{
ParserExpressionList parser(false);
ASTPtr ast = parseQuery(parser, query, 0, 0);
return ComparisonGraph(ast->children);
return ComparisonGraph<ASTPtr>(ast->children);
}
TEST(ComparisonGraph, Bounds)
@ -47,8 +47,8 @@ TEST(ComparisonGraph, Bounds)
auto x = std::make_shared<ASTIdentifier>("x");
auto y = std::make_shared<ASTIdentifier>("y");
ASSERT_EQ(graph.compare(x, y), ComparisonGraph::CompareResult::LESS);
ASSERT_EQ(graph.compare(y, x), ComparisonGraph::CompareResult::GREATER);
ASSERT_EQ(graph.compare(x, y), ComparisonGraphCompareResult::LESS);
ASSERT_EQ(graph.compare(y, x), ComparisonGraphCompareResult::GREATER);
}
}
@ -93,7 +93,7 @@ TEST(ComparisonGraph, Components)
TEST(ComparisonGraph, Compare)
{
using CompareResult = ComparisonGraph::CompareResult;
using enum ComparisonGraphCompareResult;
{
String query = "a >= b, c >= b";
@ -102,7 +102,7 @@ TEST(ComparisonGraph, Compare)
auto a = std::make_shared<ASTIdentifier>("a");
auto c = std::make_shared<ASTIdentifier>("c");
ASSERT_EQ(graph.compare(a, c), CompareResult::UNKNOWN);
ASSERT_EQ(graph.compare(a, c), UNKNOWN);
}
{
@ -113,9 +113,9 @@ TEST(ComparisonGraph, Compare)
auto b = std::make_shared<ASTIdentifier>("b");
auto c = std::make_shared<ASTIdentifier>("c");
ASSERT_EQ(graph.compare(a, c), CompareResult::GREATER);
ASSERT_EQ(graph.compare(a, b), CompareResult::GREATER_OR_EQUAL);
ASSERT_EQ(graph.compare(b, c), CompareResult::GREATER);
ASSERT_EQ(graph.compare(a, c), GREATER);
ASSERT_EQ(graph.compare(a, b), GREATER_OR_EQUAL);
ASSERT_EQ(graph.compare(b, c), GREATER);
}
{
@ -126,9 +126,9 @@ TEST(ComparisonGraph, Compare)
auto b = std::make_shared<ASTIdentifier>("b");
auto c = std::make_shared<ASTIdentifier>("c");
ASSERT_EQ(graph.compare(a, b), CompareResult::NOT_EQUAL);
ASSERT_EQ(graph.compare(a, c), CompareResult::GREATER);
ASSERT_EQ(graph.compare(b, c), CompareResult::UNKNOWN);
ASSERT_EQ(graph.compare(a, b), NOT_EQUAL);
ASSERT_EQ(graph.compare(a, c), GREATER);
ASSERT_EQ(graph.compare(b, c), UNKNOWN);
}
{
@ -154,17 +154,17 @@ TEST(ComparisonGraph, Compare)
auto lit_3 = std::make_shared<ASTLiteral>(3u);
auto lit_4 = std::make_shared<ASTLiteral>(4u);
ASSERT_EQ(graph.compare(lit_3, a), CompareResult::LESS_OR_EQUAL);
ASSERT_FALSE(graph.isAlwaysCompare(CompareResult::LESS, lit_3, a));
ASSERT_TRUE(graph.isAlwaysCompare(CompareResult::LESS, lit_2, a));
ASSERT_EQ(graph.compare(lit_3, a), LESS_OR_EQUAL);
ASSERT_FALSE(graph.isAlwaysCompare(LESS, lit_3, a));
ASSERT_TRUE(graph.isAlwaysCompare(LESS, lit_2, a));
ASSERT_EQ(graph.compare(b, lit_2), CompareResult::GREATER);
ASSERT_EQ(graph.compare(b, lit_3), CompareResult::GREATER);
ASSERT_EQ(graph.compare(b, lit_4), CompareResult::UNKNOWN);
ASSERT_EQ(graph.compare(b, lit_2), GREATER);
ASSERT_EQ(graph.compare(b, lit_3), GREATER);
ASSERT_EQ(graph.compare(b, lit_4), UNKNOWN);
ASSERT_EQ(graph.compare(d, lit_2), CompareResult::GREATER);
ASSERT_EQ(graph.compare(d, lit_3), CompareResult::GREATER_OR_EQUAL);
ASSERT_EQ(graph.compare(d, lit_4), CompareResult::UNKNOWN);
ASSERT_EQ(graph.compare(d, lit_2), GREATER);
ASSERT_EQ(graph.compare(d, lit_3), GREATER_OR_EQUAL);
ASSERT_EQ(graph.compare(d, lit_4), UNKNOWN);
}
{
@ -176,8 +176,8 @@ TEST(ComparisonGraph, Compare)
auto lit_3 = std::make_shared<ASTLiteral>(3);
auto lit_15 = std::make_shared<ASTLiteral>(15);
ASSERT_EQ(graph.compare(a, lit_8), CompareResult::UNKNOWN);
ASSERT_EQ(graph.compare(a, lit_3), CompareResult::GREATER);
ASSERT_EQ(graph.compare(a, lit_15), CompareResult::LESS);
ASSERT_EQ(graph.compare(a, lit_8), UNKNOWN);
ASSERT_EQ(graph.compare(a, lit_3), GREATER);
ASSERT_EQ(graph.compare(a, lit_15), LESS);
}
}

5
src/Parsers/ASTSystemQuery.cpp
View File

@ -182,8 +182,9 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &,
else if (!disk.empty())
print_identifier(disk);
if (strict_sync)
settings.ostr << (settings.hilite ? hilite_keyword : "") << " STRICT" << (settings.hilite ? hilite_none : "");
if (sync_replica_mode != SyncReplicaMode::DEFAULT)
settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << magic_enum::enum_name(sync_replica_mode)
<< (settings.hilite ? hilite_none : "");
}
else if (type == Type::SYNC_DATABASE_REPLICA)
{

3
src/Parsers/ASTSystemQuery.h
View File

@ -2,6 +2,7 @@
#include <Parsers/ASTQueryWithOnCluster.h>
#include <Parsers/IAST.h>
#include <Parsers/SyncReplicaMode.h>
#include "config.h"
@ -108,7 +109,7 @@ public:
String schema_cache_storage;
bool strict_sync = false;
SyncReplicaMode sync_replica_mode = SyncReplicaMode::DEFAULT;
String getID(char) const override { return "SYSTEM query"; }

11
src/Parsers/ParserSystemQuery.cpp
View File

@ -259,8 +259,15 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &
return false;
if (!parseDatabaseAndTableAsAST(pos, expected, res->database, res->table))
return false;
if (res->type == Type::SYNC_REPLICA && ParserKeyword{"STRICT"}.ignore(pos, expected))
res->strict_sync = true;
if (res->type == Type::SYNC_REPLICA)
{
if (ParserKeyword{"STRICT"}.ignore(pos, expected))
res->sync_replica_mode = SyncReplicaMode::STRICT;
else if (ParserKeyword{"LIGHTWEIGHT"}.ignore(pos, expected))
res->sync_replica_mode = SyncReplicaMode::LIGHTWEIGHT;
else if (ParserKeyword{"PULL"}.ignore(pos, expected))
res->sync_replica_mode = SyncReplicaMode::PULL;
}
break;
}

13
src/Parsers/SyncReplicaMode.h Normal file
View File

@ -0,0 +1,13 @@
#pragma once
#include <cstdint>
namespace DB
{
enum class SyncReplicaMode : uint8_t
{
DEFAULT,
STRICT,
LIGHTWEIGHT,
PULL,
};
}

13
src/Planner/Planner.cpp
View File

@ -1130,6 +1130,19 @@ void Planner::buildPlanForQueryNode()
collectSets(query_tree, *planner_context);
collectTableExpressionData(query_tree, planner_context);
const auto & settings = query_context->getSettingsRef();
if (planner_context->getTableExpressionNodeToData().size() > 1
&& (!settings.parallel_replicas_custom_key.value.empty() || settings.allow_experimental_parallel_reading_from_replicas))
{
LOG_WARNING(
&Poco::Logger::get("Planner"), "Joins are not supported with parallel replicas. Query will be executed without using them.");
auto & mutable_context = planner_context->getMutableQueryContext();
mutable_context->setSetting("allow_experimental_parallel_reading_from_replicas", false);
mutable_context->setSetting("parallel_replicas_custom_key", String{""});
}
auto top_level_identifiers = collectTopLevelColumnIdentifiers(query_tree, planner_context);
auto join_tree_query_plan = buildJoinTreeQueryPlan(query_tree,
select_query_info,

163
src/Planner/PlannerJoinTree.cpp
View File

@ -17,6 +17,7 @@
#include <Storages/IStorage.h>
#include <Storages/StorageDictionary.h>
#include <Storages/StorageDistributed.h>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/ColumnNode.h>
@ -47,6 +48,7 @@
#include <Interpreters/TableJoin.h>
#include <Interpreters/HashJoin.h>
#include <Interpreters/ArrayJoinAction.h>
#include <Interpreters/getCustomKeyFilterForParallelReplicas.h>
#include <Planner/CollectColumnIdentifiers.h>
#include <Planner/Planner.h>
@ -381,6 +383,46 @@ void updatePrewhereOutputsIfNeeded(SelectQueryInfo & table_expression_query_info
prewhere_outputs.insert(prewhere_outputs.end(), required_output_nodes.begin(), required_output_nodes.end());
}
FilterDAGInfo buildFilterInfo(ASTPtr filter_expression,
SelectQueryInfo & table_expression_query_info,
PlannerContextPtr & planner_context)
{
const auto & query_context = planner_context->getQueryContext();
auto filter_query_tree = buildQueryTree(filter_expression, query_context);
QueryAnalysisPass query_analysis_pass(table_expression_query_info.table_expression);
query_analysis_pass.run(filter_query_tree, query_context);
auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression_query_info.table_expression);
const auto table_expression_names = table_expression_data.getColumnNames();
NameSet table_expression_required_names_without_filter(table_expression_names.begin(), table_expression_names.end());
collectSourceColumns(filter_query_tree, planner_context);
collectSets(filter_query_tree, *planner_context);
auto filter_actions_dag = std::make_shared<ActionsDAG>();
PlannerActionsVisitor actions_visitor(planner_context, false /*use_column_identifier_as_action_node_name*/);
auto expression_nodes = actions_visitor.visit(filter_actions_dag, filter_query_tree);
if (expression_nodes.size() != 1)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Filter actions must return single output node. Actual {}",
expression_nodes.size());
auto & filter_actions_outputs = filter_actions_dag->getOutputs();
filter_actions_outputs = std::move(expression_nodes);
std::string filter_node_name = filter_actions_outputs[0]->result_name;
bool remove_filter_column = true;
for (const auto & filter_input_node : filter_actions_dag->getInputs())
if (table_expression_required_names_without_filter.contains(filter_input_node->result_name))
filter_actions_outputs.push_back(filter_input_node);
return {std::move(filter_actions_dag), std::move(filter_node_name), remove_filter_column};
}
FilterDAGInfo buildRowPolicyFilterIfNeeded(const StoragePtr & storage,
SelectQueryInfo & table_expression_query_info,
PlannerContextPtr & planner_context)
@ -392,38 +434,38 @@ FilterDAGInfo buildRowPolicyFilterIfNeeded(const StoragePtr & storage,
if (!row_policy_filter)
return {};
auto row_policy_filter_query_tree = buildQueryTree(row_policy_filter->expression, query_context);
return buildFilterInfo(row_policy_filter->expression, table_expression_query_info, planner_context);
}
QueryAnalysisPass query_analysis_pass(table_expression_query_info.table_expression);
query_analysis_pass.run(row_policy_filter_query_tree, query_context);
FilterDAGInfo buildCustomKeyFilterIfNeeded(const StoragePtr & storage,
SelectQueryInfo & table_expression_query_info,
PlannerContextPtr & planner_context)
{
const auto & query_context = planner_context->getQueryContext();
const auto & settings = query_context->getSettingsRef();
auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression_query_info.table_expression);
const auto table_expression_names = table_expression_data.getColumnNames();
NameSet table_expression_required_names_without_row_policy(table_expression_names.begin(), table_expression_names.end());
if (settings.parallel_replicas_count <= 1 || settings.parallel_replicas_custom_key.value.empty())
return {};
collectSourceColumns(row_policy_filter_query_tree, planner_context);
collectSets(row_policy_filter_query_tree, *planner_context);
auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *query_context);
if (!custom_key_ast)
throw DB::Exception(
ErrorCodes::BAD_ARGUMENTS,
"Parallel replicas processing with custom_key has been requested "
"(setting 'max_parallel_replcias'), but the table does not have custom_key defined for it "
" or it's invalid (setting 'parallel_replicas_custom_key')");
auto row_policy_actions_dag = std::make_shared<ActionsDAG>();
LOG_TRACE(&Poco::Logger::get("Planner"), "Processing query on a replica using custom_key '{}'", settings.parallel_replicas_custom_key.value);
PlannerActionsVisitor actions_visitor(planner_context, false /*use_column_identifier_as_action_node_name*/);
auto expression_nodes = actions_visitor.visit(row_policy_actions_dag, row_policy_filter_query_tree);
if (expression_nodes.size() != 1)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Row policy filter actions must return single output node. Actual {}",
expression_nodes.size());
auto parallel_replicas_custom_filter_ast = getCustomKeyFilterForParallelReplica(
settings.parallel_replicas_count,
settings.parallel_replica_offset,
std::move(custom_key_ast),
settings.parallel_replicas_custom_key_filter_type,
*storage,
query_context);
auto & row_policy_actions_outputs = row_policy_actions_dag->getOutputs();
row_policy_actions_outputs = std::move(expression_nodes);
std::string filter_node_name = row_policy_actions_outputs[0]->result_name;
bool remove_filter_column = true;
for (const auto & row_policy_input_node : row_policy_actions_dag->getInputs())
if (table_expression_required_names_without_row_policy.contains(row_policy_input_node->result_name))
row_policy_actions_outputs.push_back(row_policy_input_node);
return {std::move(row_policy_actions_dag), std::move(filter_node_name), remove_filter_column};
return buildFilterInfo(parallel_replicas_custom_filter_ast, table_expression_query_info, planner_context);
}
JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expression,
@ -596,11 +638,14 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
updatePrewhereOutputsIfNeeded(table_expression_query_info, table_expression_data.getColumnNames(), storage_snapshot);
auto row_policy_filter_info = buildRowPolicyFilterIfNeeded(storage, table_expression_query_info, planner_context);
bool moved_row_policy_to_prewhere = false;
const auto & columns_names = table_expression_data.getColumnNames();
if (row_policy_filter_info.actions)
std::vector<std::pair<FilterDAGInfo, std::string>> where_filters;
const auto add_filter = [&](const FilterDAGInfo & filter_info, std::string description)
{
if (!filter_info.actions)
return;
bool is_final = table_expression_query_info.table_expression_modifiers &&
table_expression_query_info.table_expression_modifiers->hasFinal();
bool optimize_move_to_prewhere = settings.optimize_move_to_prewhere && (!is_final || settings.optimize_move_to_prewhere_if_final);
@ -612,36 +657,62 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
if (!table_expression_query_info.prewhere_info->prewhere_actions)
{
table_expression_query_info.prewhere_info->prewhere_actions = row_policy_filter_info.actions;
table_expression_query_info.prewhere_info->prewhere_column_name = row_policy_filter_info.column_name;
table_expression_query_info.prewhere_info->remove_prewhere_column = row_policy_filter_info.do_remove_column;
table_expression_query_info.prewhere_info->prewhere_actions = filter_info.actions;
table_expression_query_info.prewhere_info->prewhere_column_name = filter_info.column_name;
table_expression_query_info.prewhere_info->remove_prewhere_column = filter_info.do_remove_column;
}
else
{
table_expression_query_info.prewhere_info->row_level_filter = row_policy_filter_info.actions;
table_expression_query_info.prewhere_info->row_level_column_name = row_policy_filter_info.column_name;
table_expression_query_info.prewhere_info->row_level_filter = filter_info.actions;
table_expression_query_info.prewhere_info->row_level_column_name = filter_info.column_name;
}
table_expression_query_info.prewhere_info->need_filter = true;
moved_row_policy_to_prewhere = true;
}
else
{
where_filters.emplace_back(filter_info, std::move(description));
}
};
auto row_policy_filter_info = buildRowPolicyFilterIfNeeded(storage, table_expression_query_info, planner_context);
add_filter(row_policy_filter_info, "Row-level security filter");
if (query_context->getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY)
{
if (settings.parallel_replicas_count > 1)
{
auto parallel_replicas_custom_key_filter_info = buildCustomKeyFilterIfNeeded(storage, table_expression_query_info, planner_context);
add_filter(parallel_replicas_custom_key_filter_info, "Parallel replicas custom key filter");
}
else
{
if (auto * distributed = typeid_cast<StorageDistributed *>(storage.get());
distributed && canUseCustomKey(settings, *distributed->getCluster(), *query_context))
{
table_expression_query_info.use_custom_key = true;
planner_context->getMutableQueryContext()->setSetting("distributed_group_by_no_merge", 2);
}
}
}
const auto & columns_names = table_expression_data.getColumnNames();
from_stage = storage->getQueryProcessingStage(query_context, select_query_options.to_stage, storage_snapshot, table_expression_query_info);
storage->read(query_plan, columns_names, storage_snapshot, table_expression_query_info, query_context, from_stage, max_block_size, max_streams);
if (query_plan.isInitialized() &&
from_stage == QueryProcessingStage::FetchColumns &&
row_policy_filter_info.actions &&
!moved_row_policy_to_prewhere)
for (const auto & filter_info_and_description : where_filters)
{
auto row_level_filter_step = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(),
row_policy_filter_info.actions,
row_policy_filter_info.column_name,
row_policy_filter_info.do_remove_column);
row_level_filter_step->setStepDescription("Row-level security filter");
query_plan.addStep(std::move(row_level_filter_step));
const auto & [filter_info, description] = filter_info_and_description;
if (query_plan.isInitialized() &&
from_stage == QueryProcessingStage::FetchColumns &&
filter_info.actions)
{
auto filter_step = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(),
filter_info.actions,
filter_info.column_name,
filter_info.do_remove_column);
filter_step->setStepDescription(description);
query_plan.addStep(std::move(filter_step));
}
}
if (query_context->hasQueryContext() && !select_query_options.is_internal)

6
src/Processors/examples/CMakeLists.txt
View File

@ -1,2 +1,4 @@
clickhouse_add_executable (comma_separated_streams comma_separated_streams.cpp)
target_link_libraries (comma_separated_streams PRIVATE dbms)
if (TARGET ch_contrib::hivemetastore)
clickhouse_add_executable (comma_separated_streams comma_separated_streams.cpp)
target_link_libraries (comma_separated_streams PRIVATE dbms)
endif()

2
src/Server/PrometheusMetricsWriter.cpp
View File

@ -59,7 +59,7 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const
{
if (send_events)
{
for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
for (ProfileEvents::Event i = ProfileEvents::Event(0), end = ProfileEvents::end(); i < end; ++i)
{
const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed);

3
src/Server/TCPHandler.cpp
View File

@ -1202,7 +1202,8 @@ void TCPHandler::receiveHello()
throw Exception(ErrorCodes::CLIENT_HAS_CONNECTED_TO_WRONG_PORT, "Client has connected to wrong port");
}
else
throw NetException(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet from client");
throw NetException(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT,
"Unexpected packet from client (expected Hello, got {})", packet_type);
}
readStringBinary(client_name, *in);

103
src/Storages/ConstraintsDescription.cpp
View File

@ -11,6 +11,13 @@
#include <Core/Defines.h>
#include <Analyzer/QueryTreeBuilder.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/TableNode.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/Passes/QueryAnalysisPass.h>
#include <Interpreters/Context.h>
namespace DB
{
@ -103,7 +110,7 @@ std::vector<CNFQuery::AtomicFormula> ConstraintsDescription::getAtomicConstraint
return constraint_data;
}
std::unique_ptr<ComparisonGraph> ConstraintsDescription::buildGraph() const
std::unique_ptr<ComparisonGraph<ASTPtr>> ConstraintsDescription::buildGraph() const
{
static const NameSet relations = { "equals", "less", "lessOrEquals", "greaterOrEquals", "greater" };
@ -121,7 +128,7 @@ std::unique_ptr<ComparisonGraph> ConstraintsDescription::buildGraph() const
}
}
return std::make_unique<ComparisonGraph>(constraints_for_graph);
return std::make_unique<ComparisonGraph<ASTPtr>>(constraints_for_graph);
}
ConstraintsExpressions ConstraintsDescription::getExpressions(const DB::ContextPtr context,
@ -143,7 +150,7 @@ ConstraintsExpressions ConstraintsDescription::getExpressions(const DB::ContextP
return res;
}
const ComparisonGraph & ConstraintsDescription::getGraph() const
const ComparisonGraph<ASTPtr> & ConstraintsDescription::getGraph() const
{
return *graph;
}
@ -175,6 +182,94 @@ std::vector<CNFQuery::AtomicFormula> ConstraintsDescription::getAtomsById(const
return result;
}
ConstraintsDescription::QueryTreeData ConstraintsDescription::getQueryTreeData(const ContextPtr & context, const QueryTreeNodePtr & table_node) const
{
QueryTreeData data;
std::vector<Analyzer::CNF::AtomicFormula> atomic_constraints_data;
QueryAnalysisPass pass(table_node);
for (const auto & constraint : filterConstraints(ConstraintsDescription::ConstraintType::ALWAYS_TRUE))
{
auto query_tree = buildQueryTree(constraint->as<ASTConstraintDeclaration>()->expr->ptr(), context);
pass.run(query_tree, context);
const auto cnf = Analyzer::CNF::toCNF(query_tree, context)
.pullNotOutFunctions(context);
for (const auto & group : cnf.getStatements())
{
data.cnf_constraints.emplace_back(group.begin(), group.end());
if (group.size() == 1)
atomic_constraints_data.emplace_back(*group.begin());
}
data.constraints.push_back(std::move(query_tree));
}
for (size_t i = 0; i < data.cnf_constraints.size(); ++i)
for (size_t j = 0; j < data.cnf_constraints[i].size(); ++j)
data.query_node_to_atom_ids[data.cnf_constraints[i][j].node_with_hash].push_back({i, j});
/// build graph
if (constraints.empty())
{
data.graph = std::make_unique<ComparisonGraph<QueryTreeNodePtr>>(QueryTreeNodes(), context);
}
else
{
static const NameSet relations = { "equals", "less", "lessOrEquals", "greaterOrEquals", "greater" };
QueryTreeNodes constraints_for_graph;
for (const auto & atomic_formula : atomic_constraints_data)
{
Analyzer::CNF::AtomicFormula atom{atomic_formula.negative, atomic_formula.node_with_hash.node->clone()};
atom = Analyzer::CNF::pushNotIntoFunction(atom, context);
auto * function_node = atom.node_with_hash.node->as<FunctionNode>();
if (function_node && relations.contains(function_node->getFunctionName()))
{
assert(!atom.negative);
constraints_for_graph.push_back(atom.node_with_hash.node);
}
}
data.graph = std::make_unique<ComparisonGraph<QueryTreeNodePtr>>(constraints_for_graph, context);
}
return data;
}
const QueryTreeNodes & ConstraintsDescription::QueryTreeData::getConstraints() const
{
return constraints;
}
const std::vector<std::vector<Analyzer::CNF::AtomicFormula>> & ConstraintsDescription::QueryTreeData::getConstraintData() const
{
return cnf_constraints;
}
const ComparisonGraph<QueryTreeNodePtr> & ConstraintsDescription::QueryTreeData::getGraph() const
{
return *graph;
}
std::optional<ConstraintsDescription::AtomIds> ConstraintsDescription::QueryTreeData::getAtomIds(const QueryTreeNodePtrWithHash & node_with_hash) const
{
auto it = query_node_to_atom_ids.find(node_with_hash);
if (it != query_node_to_atom_ids.end())
return it->second;
return std::nullopt;
}
std::vector<Analyzer::CNF::AtomicFormula> ConstraintsDescription::QueryTreeData::getAtomsById(const AtomIds & ids) const
{
std::vector<Analyzer::CNF::AtomicFormula> result;
for (const auto & id : ids)
result.push_back(cnf_constraints[id.group_id][id.atom_id]);
return result;
}
ConstraintsDescription::ConstraintsDescription(const ASTs & constraints_)
: constraints(constraints_)
{
@ -218,7 +313,7 @@ void ConstraintsDescription::update()
{
cnf_constraints.clear();
ast_to_atom_ids.clear();
graph = std::make_unique<ComparisonGraph>(ASTs());
graph = std::make_unique<ComparisonGraph<ASTPtr>>(ASTs());
return;
}

29
src/Storages/ConstraintsDescription.h
View File

@ -5,6 +5,8 @@
#include <Interpreters/TreeCNFConverter.h>
#include <Interpreters/ComparisonGraph.h>
#include <Analyzer/Passes/CNF.h>
namespace DB
{
@ -41,7 +43,7 @@ public:
const std::vector<std::vector<CNFQuery::AtomicFormula>> & getConstraintData() const;
std::vector<CNFQuery::AtomicFormula> getAtomicConstraintData() const;
const ComparisonGraph & getGraph() const;
const ComparisonGraph<ASTPtr> & getGraph() const;
ConstraintsExpressions getExpressions(ContextPtr context, const NamesAndTypesList & source_columns_) const;
@ -56,15 +58,36 @@ public:
std::optional<AtomIds> getAtomIds(const ASTPtr & ast) const;
std::vector<CNFQuery::AtomicFormula> getAtomsById(const AtomIds & ids) const;
class QueryTreeData
{
public:
const QueryTreeNodes & getConstraints() const;
const std::vector<std::vector<Analyzer::CNF::AtomicFormula>> & getConstraintData() const;
std::optional<AtomIds> getAtomIds(const QueryTreeNodePtrWithHash & node_with_hash) const;
std::vector<Analyzer::CNF::AtomicFormula> getAtomsById(const AtomIds & ids) const;
const ComparisonGraph<QueryTreeNodePtr> & getGraph() const;
private:
QueryTreeNodes constraints;
std::vector<std::vector<Analyzer::CNF::AtomicFormula>> cnf_constraints;
QueryTreeNodePtrWithHashMap<AtomIds> query_node_to_atom_ids;
std::unique_ptr<ComparisonGraph<QueryTreeNodePtr>> graph;
friend ConstraintsDescription;
};
QueryTreeData getQueryTreeData(const ContextPtr & context, const QueryTreeNodePtr & table_node) const;
private:
std::vector<std::vector<CNFQuery::AtomicFormula>> buildConstraintData() const;
std::unique_ptr<ComparisonGraph> buildGraph() const;
std::unique_ptr<ComparisonGraph<ASTPtr>> buildGraph() const;
void update();
ASTs constraints;
std::vector<std::vector<CNFQuery::AtomicFormula>> cnf_constraints;
std::map<IAST::Hash, AtomIds> ast_to_atom_ids;
std::unique_ptr<ComparisonGraph> graph;
std::unique_ptr<ComparisonGraph<ASTPtr>> graph;
};
}

9
src/Storages/Distributed/DistributedSink.cpp
View File

@ -41,6 +41,8 @@
namespace CurrentMetrics
{
extern const Metric DistributedSend;
extern const Metric DistributedInsertThreads;
extern const Metric DistributedInsertThreadsActive;
}
namespace ProfileEvents
@ -460,9 +462,10 @@ void DistributedSink::writeSync(const Block & block)
size_t jobs_count = random_shard_insert ? 1 : (remote_jobs_count + local_jobs_count);
size_t max_threads = std::min<size_t>(settings.max_distributed_connections, jobs_count);
pool.emplace(/* max_threads_= */ max_threads,
/* max_free_threads_= */ max_threads,
/* queue_size_= */ jobs_count);
pool.emplace(
CurrentMetrics::DistributedInsertThreads,
CurrentMetrics::DistributedInsertThreadsActive,
max_threads, max_threads, jobs_count);
if (!throttler && (settings.max_network_bandwidth || settings.max_network_bytes))
{

12
src/Storages/MergeTree/MergeTreeIndexHypothesisMergedCondition.cpp
View File

@ -108,7 +108,7 @@ bool MergeTreeIndexhypothesisMergedCondition::alwaysUnknownOrTrue() const
func->name = "greaterOrEquals";
}
const auto weak_graph = std::make_unique<ComparisonGraph>(active_atomic_formulas);
const auto weak_graph = std::make_unique<ComparisonGraph<ASTPtr>>(active_atomic_formulas);
bool useless = true;
expression_cnf->iterateGroups(
@ -146,7 +146,7 @@ bool MergeTreeIndexhypothesisMergedCondition::mayBeTrueOnGranule(const MergeTree
values.push_back(granule->met);
}
const ComparisonGraph * graph = nullptr;
const ComparisonGraph<ASTPtr> * graph = nullptr;
{
std::lock_guard lock(cache_mutex);
@ -170,7 +170,7 @@ bool MergeTreeIndexhypothesisMergedCondition::mayBeTrueOnGranule(const MergeTree
const auto * func = atom.ast->as<ASTFunction>();
if (func && func->arguments->children.size() == 2)
{
const auto expected = ComparisonGraph::atomToCompareResult(atom);
const auto expected = ComparisonGraph<ASTPtr>::atomToCompareResult(atom);
if (graph->isPossibleCompare(expected, func->arguments->children[0], func->arguments->children[1]))
{
/// If graph failed use matching.
@ -188,7 +188,7 @@ bool MergeTreeIndexhypothesisMergedCondition::mayBeTrueOnGranule(const MergeTree
return !always_false;
}
std::unique_ptr<ComparisonGraph> MergeTreeIndexhypothesisMergedCondition::buildGraph(const std::vector<bool> & values) const
std::unique_ptr<ComparisonGraph<ASTPtr>> MergeTreeIndexhypothesisMergedCondition::buildGraph(const std::vector<bool> & values) const
{
ASTs active_atomic_formulas(atomic_constraints);
for (size_t i = 0; i < values.size(); ++i)
@ -199,10 +199,10 @@ std::unique_ptr<ComparisonGraph> MergeTreeIndexhypothesisMergedCondition::buildG
std::begin(index_to_compare_atomic_hypotheses[i]),
std::end(index_to_compare_atomic_hypotheses[i]));
}
return std::make_unique<ComparisonGraph>(active_atomic_formulas);
return std::make_unique<ComparisonGraph<ASTPtr>>(active_atomic_formulas);
}
const ComparisonGraph * MergeTreeIndexhypothesisMergedCondition::getGraph(const std::vector<bool> & values) const
const ComparisonGraph<ASTPtr> * MergeTreeIndexhypothesisMergedCondition::getGraph(const std::vector<bool> & values) const
{
auto [it, inserted] = graph_cache.try_emplace(values);
if (inserted)

6
src/Storages/MergeTree/MergeTreeIndexHypothesisMergedCondition.h
View File

@ -20,8 +20,8 @@ public:
private:
void addConstraints(const ConstraintsDescription & constraints_description);
std::unique_ptr<ComparisonGraph> buildGraph(const std::vector<bool> & values) const;
const ComparisonGraph * getGraph(const std::vector<bool> & values) const;
std::unique_ptr<ComparisonGraph<ASTPtr>> buildGraph(const std::vector<bool> & values) const;
const ComparisonGraph<ASTPtr> * getGraph(const std::vector<bool> & values) const;
ASTPtr expression_ast;
std::unique_ptr<CNFQuery> expression_cnf;
@ -29,7 +29,7 @@ private:
/// Part analysis can be done in parallel.
/// So, we have shared answer and graph cache.
mutable std::mutex cache_mutex;
mutable std::unordered_map<std::vector<bool>, std::unique_ptr<ComparisonGraph>> graph_cache;
mutable std::unordered_map<std::vector<bool>, std::unique_ptr<ComparisonGraph<ASTPtr>>> graph_cache;
mutable std::unordered_map<std::vector<bool>, bool> answer_cache;
std::vector<std::vector<ASTPtr>> index_to_compare_atomic_hypotheses;

58
src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
View File

@ -544,7 +544,7 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep
if (!found && need_remove_from_zk)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't find {} in the memory queue. It is a bug. Entry: {}",
entry->znode_name, entry->toString());
notifySubscribers(queue_size, entry->znode_name);
notifySubscribers(queue_size, &(entry->znode_name));
if (!need_remove_from_zk)
return;
@ -2463,40 +2463,64 @@ String ReplicatedMergeTreeMergePredicate::getCoveringVirtualPart(const String &
ReplicatedMergeTreeQueue::SubscriberHandler
ReplicatedMergeTreeQueue::addSubscriber(ReplicatedMergeTreeQueue::SubscriberCallBack && callback)
ReplicatedMergeTreeQueue::addSubscriber(ReplicatedMergeTreeQueue::SubscriberCallBack && callback,
std::unordered_set<String> & out_entry_names, SyncReplicaMode sync_mode)
{
std::lock_guard lock(state_mutex);
std::unordered_set<String> result;
result.reserve(queue.size());
for (const auto & entry : queue)
result.insert(entry->znode_name);
std::lock_guard<std::mutex> lock(state_mutex);
std::lock_guard lock_subscribers(subscribers_mutex);
if (sync_mode != SyncReplicaMode::PULL)
{
/// We must get the list of entries to wait atomically with adding the callback
bool lightweight_entries_only = sync_mode == SyncReplicaMode::LIGHTWEIGHT;
static constexpr std::array lightweight_entries =
{
LogEntry::GET_PART,
LogEntry::ATTACH_PART,
LogEntry::DROP_RANGE,
LogEntry::REPLACE_RANGE,
LogEntry::DROP_PART
};
out_entry_names.reserve(queue.size());
for (const auto & entry : queue)
{
if (!lightweight_entries_only
|| std::find(lightweight_entries.begin(), lightweight_entries.end(), entry->type) != lightweight_entries.end())
out_entry_names.insert(entry->znode_name);
}
}
auto it = subscribers.emplace(subscribers.end(), std::move(callback));
/// Notify queue size & log entry ids to avoid waiting for removed entries
(*it)(result.size(), result, std::nullopt);
/// Atomically notify about current size
(*it)(queue.size(), nullptr);
return SubscriberHandler(it, *this);
}
void ReplicatedMergeTreeQueue::notifySubscribersOnPartialShutdown()
{
size_t queue_size;
{
std::lock_guard<std::mutex> lock(state_mutex);
queue_size = queue.size();
}
std::lock_guard lock_subscribers(subscribers_mutex);
for (auto & subscriber_callback : subscribers)
subscriber_callback(queue_size, nullptr);
}
ReplicatedMergeTreeQueue::SubscriberHandler::~SubscriberHandler()
{
std::lock_guard lock(queue.subscribers_mutex);
queue.subscribers.erase(it);
}
void ReplicatedMergeTreeQueue::notifySubscribers(size_t new_queue_size, std::optional<String> removed_log_entry_id)
void ReplicatedMergeTreeQueue::notifySubscribers(size_t new_queue_size, const String * removed_log_entry_id)
{
std::lock_guard lock_subscribers(subscribers_mutex);
for (auto & subscriber_callback : subscribers)
subscriber_callback(new_queue_size, {}, removed_log_entry_id);
}
ReplicatedMergeTreeQueue::~ReplicatedMergeTreeQueue()
{
notifySubscribers(0, std::nullopt);
subscriber_callback(new_queue_size, removed_log_entry_id);
}
String padIndex(Int64 index)

11
src/Storages/MergeTree/ReplicatedMergeTreeQueue.h
View File

@ -3,6 +3,7 @@
#include <optional>
#include <Common/ActionBlocker.h>
#include <Parsers/SyncReplicaMode.h>
#include <Storages/MergeTree/ReplicatedMergeTreeLogEntry.h>
#include <Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h>
#include <Storages/MergeTree/ActiveDataPartSet.h>
@ -164,7 +165,7 @@ private:
/// A subscriber callback is called when an entry queue is deleted
mutable std::mutex subscribers_mutex;
using SubscriberCallBack = std::function<void(size_t /* queue_size */, std::unordered_set<String> /*wait_for_ids*/, std::optional<String> /* removed_log_entry_id */)>;
using SubscriberCallBack = std::function<void(size_t /* queue_size */, const String * /* removed_log_entry_id */)>;
using Subscribers = std::list<SubscriberCallBack>;
using SubscriberIterator = Subscribers::iterator;
@ -182,7 +183,7 @@ private:
Subscribers subscribers;
/// Notify subscribers about queue change (new queue size and entry that was removed)
void notifySubscribers(size_t new_queue_size, std::optional<String> removed_log_entry_id);
void notifySubscribers(size_t new_queue_size, const String * removed_log_entry_id);
/// Check that entry_ptr is REPLACE_RANGE entry and can be removed from queue because current entry covers it
bool checkReplaceRangeCanBeRemoved(
@ -287,7 +288,7 @@ private:
public:
ReplicatedMergeTreeQueue(StorageReplicatedMergeTree & storage_, ReplicatedMergeTreeMergeStrategyPicker & merge_strategy_picker_);
~ReplicatedMergeTreeQueue();
~ReplicatedMergeTreeQueue() = default;
/// Clears queue state
void clear();
@ -425,7 +426,9 @@ public:
ActionBlocker pull_log_blocker;
/// Adds a subscriber
SubscriberHandler addSubscriber(SubscriberCallBack && callback);
SubscriberHandler addSubscriber(SubscriberCallBack && callback, std::unordered_set<String> & out_entry_names, SyncReplicaMode sync_mode);
void notifySubscribersOnPartialShutdown();
struct Status
{

2
src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
View File

@ -250,7 +250,7 @@ void StorageMaterializedPostgreSQL::dropInnerTableIfAny(bool sync, ContextPtr lo
auto nested_table = tryGetNested() != nullptr;
if (nested_table)
InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, getNestedStorageID(), sync);
InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, getNestedStorageID(), sync, /* ignore_sync_setting */ true);
}

17
src/Storages/StorageMaterializedView.cpp
View File

@ -212,13 +212,26 @@ void StorageMaterializedView::drop()
if (!select_query.select_table_id.empty())
DatabaseCatalog::instance().removeViewDependency(select_query.select_table_id, table_id);
dropInnerTableIfAny(true, getContext());
/// Sync flag and the setting make sense for Atomic databases only.
/// However, with Atomic databases, IStorage::drop() can be called only from a background task in DatabaseCatalog.
/// Running synchronous DROP from that task leads to deadlock.
/// Usually dropInnerTableIfAny is no-op, because the inner table is dropped before enqueueing a drop task for the MV itself.
/// But there's a race condition with SYSTEM RESTART REPLICA: the inner table might be detached due to RESTART.
/// In this case, dropInnerTableIfAny will not find the inner table and will not drop it during executions of DROP query for the MV itself.
/// DDLGuard does not protect from that, because RESTART REPLICA acquires DDLGuard for the inner table name,
/// but DROP acquires DDLGuard for the name of MV. And we cannot acquire second DDLGuard for the inner name in DROP,
/// because it may lead to lock-order-inversion (DDLGuards must be acquired in lexicographical order).
dropInnerTableIfAny(/* sync */ false, getContext());
}
void StorageMaterializedView::dropInnerTableIfAny(bool sync, ContextPtr local_context)
{
/// We will use `sync` argument wneh this function is called from a DROP query
/// and will ignore database_atomic_wait_for_drop_and_detach_synchronously when it's called from drop task.
/// See the comment in StorageMaterializedView::drop
if (has_inner_table && tryGetTargetTable())
InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, sync);
InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id,
sync, /* ignore_sync_setting */ true);
}
void StorageMaterializedView::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &)

8
src/Storages/StorageMerge.cpp
View File

@ -649,14 +649,13 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources(
QueryProcessingStage::Complete,
storage_snapshot,
modified_query_info);
if (processed_stage <= storage_stage || (allow_experimental_analyzer && processed_stage == QueryProcessingStage::FetchColumns))
{
/// If there are only virtual columns in query, you must request at least one other column.
if (real_column_names.empty())
real_column_names.push_back(ExpressionActions::getSmallestColumn(storage_snapshot->metadata->getColumns().getAllPhysical()).name);
/// Steps for reading from child tables should have the same lifetime as the current step
/// because `builder` can have references to them (mainly for EXPLAIN PIPELINE).
QueryPlan & plan = child_plans.emplace_back();
StorageView * view = dynamic_cast<StorageView *>(storage.get());
@ -709,12 +708,15 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources(
modified_context->setSetting("max_threads", streams_num);
modified_context->setSetting("max_streams_to_max_threads_ratio", 1);
QueryPlan & plan = child_plans.emplace_back();
if (allow_experimental_analyzer)
{
InterpreterSelectQueryAnalyzer interpreter(modified_query_info.query_tree,
modified_context,
SelectQueryOptions(processed_stage).ignoreProjections());
builder = std::make_unique<QueryPipelineBuilder>(interpreter.buildQueryPipeline());
plan = std::move(interpreter.getPlanner()).extractQueryPlan();
}
else
{
@ -723,7 +725,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources(
InterpreterSelectQuery interpreter{modified_query_info.query,
modified_context,
SelectQueryOptions(processed_stage).ignoreProjections()};
builder = std::make_unique<QueryPipelineBuilder>(interpreter.buildQueryPipeline());
builder = std::make_unique<QueryPipelineBuilder>(interpreter.buildQueryPipeline(plan));
}
/** Materialization is needed, since from distributed storage the constants come materialized.

2
src/Storages/StorageMerge.h
View File

@ -160,7 +160,7 @@ private:
StorageSnapshotPtr merge_storage_snapshot;
/// Store read plan for each child table.
/// It's needed to guarantee lifetime for child steps to be the same as for this step.
/// It's needed to guarantee lifetime for child steps to be the same as for this step (mainly for EXPLAIN PIPELINE).
std::vector<QueryPlan> child_plans;
SelectQueryInfo query_info;

53
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -4415,6 +4415,7 @@ void StorageReplicatedMergeTree::partialShutdown()
partial_shutdown_called = true;
partial_shutdown_event.set();
queue.notifySubscribersOnPartialShutdown();
replica_is_active_node = nullptr;
LOG_TRACE(log, "Waiting for threads to finish");
@ -7578,53 +7579,55 @@ void StorageReplicatedMergeTree::onActionLockRemove(StorageActionBlockType actio
background_moves_assignee.trigger();
}
bool StorageReplicatedMergeTree::waitForProcessingQueue(UInt64 max_wait_milliseconds, bool strict)
bool StorageReplicatedMergeTree::waitForProcessingQueue(UInt64 max_wait_milliseconds, SyncReplicaMode sync_mode)
{
Stopwatch watch;
/// Let's fetch new log entries firstly
queue.pullLogsToQueue(getZooKeeperAndAssertNotReadonly(), {}, ReplicatedMergeTreeQueue::SYNC);
if (sync_mode == SyncReplicaMode::PULL)
return true;
/// This is significant, because the execution of this task could be delayed at BackgroundPool.
/// And we force it to be executed.
background_operations_assignee.trigger();
std::unordered_set<String> wait_for_ids;
bool set_ids_to_wait = true;
bool was_interrupted = false;
Poco::Event target_entry_event;
auto callback = [&target_entry_event, &wait_for_ids, &set_ids_to_wait, strict]
(size_t new_queue_size, std::unordered_set<String> log_entry_ids, std::optional<String> removed_log_entry_id)
auto callback = [this, &target_entry_event, &wait_for_ids, &was_interrupted, sync_mode]
(size_t new_queue_size, const String * removed_log_entry_id)
{
if (strict)
if (partial_shutdown_called)
{
/// In strict mode we wait for queue to become empty
was_interrupted = true;
target_entry_event.set();
return;
}
if (sync_mode == SyncReplicaMode::STRICT)
{
/// Wait for queue to become empty
if (new_queue_size == 0)
target_entry_event.set();
return;
}
if (set_ids_to_wait)
{
wait_for_ids = log_entry_ids;
set_ids_to_wait = false;
}
if (removed_log_entry_id)
wait_for_ids.erase(*removed_log_entry_id);
if (removed_log_entry_id.has_value())
wait_for_ids.erase(removed_log_entry_id.value());
if (wait_for_ids.empty() || new_queue_size == 0)
chassert(new_queue_size || wait_for_ids.empty());
if (wait_for_ids.empty())
target_entry_event.set();
};
const auto handler = queue.addSubscriber(std::move(callback));
const auto handler = queue.addSubscriber(std::move(callback), wait_for_ids, sync_mode);
while (!target_entry_event.tryWait(50))
{
if (max_wait_milliseconds && watch.elapsedMilliseconds() > max_wait_milliseconds)
return false;
if (!target_entry_event.tryWait(max_wait_milliseconds))
return false;
if (was_interrupted)
throw Exception(ErrorCodes::ABORTED, "Shutdown is called for table");
if (partial_shutdown_called)
throw Exception(ErrorCodes::ABORTED, "Shutdown is called for table");
}
return true;
}

3
src/Storages/StorageReplicatedMergeTree.h
View File

@ -38,6 +38,7 @@
#include <Core/BackgroundSchedulePool.h>
#include <QueryPipeline/Pipe.h>
#include <Storages/MergeTree/BackgroundJobsAssignee.h>
#include <Parsers/SyncReplicaMode.h>
namespace DB
@ -181,7 +182,7 @@ public:
/// Wait till replication queue's current last entry is processed or till size becomes 0
/// If timeout is exceeded returns false
bool waitForProcessingQueue(UInt64 max_wait_milliseconds, bool strict);
bool waitForProcessingQueue(UInt64 max_wait_milliseconds, SyncReplicaMode sync_mode);
/// Get the status of the table. If with_zk_fields = false - do not fill in the fields that require queries to ZK.
void getStatus(ReplicatedTableStatus & res, bool with_zk_fields = true);

2
src/Storages/System/StorageSystemEvents.cpp
View File

@ -18,7 +18,7 @@ NamesAndTypesList StorageSystemEvents::getNamesAndTypes()
void StorageSystemEvents::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const
{
for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
for (ProfileEvents::Event i = ProfileEvents::Event(0), end = ProfileEvents::end(); i < end; ++i)
{
UInt64 value = ProfileEvents::global_counters[i];

4
src/Storages/WindowView/StorageWindowView.cpp
View File

@ -1609,7 +1609,7 @@ void StorageWindowView::drop()
{
/// Must be guaranteed at this point for database engine Atomic that has_inner_table == false,
/// because otherwise will be a deadlock.
dropInnerTableIfAny(true, getContext());
dropInnerTableIfAny(false, getContext());
}
void StorageWindowView::dropInnerTableIfAny(bool sync, ContextPtr local_context)
@ -1623,7 +1623,7 @@ void StorageWindowView::dropInnerTableIfAny(bool sync, ContextPtr local_context)
ASTDropQuery::Kind::Drop, getContext(), local_context, inner_table_id, sync);
if (has_inner_target_table)
InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, sync);
InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, sync, /* ignore_sync_setting */ true);
}
catch (...)
{

7
tests/ci/stress_tests.lib
View File

@ -149,9 +149,14 @@ function stop()
if [ $check_hang == true ]
then
# We failed to stop the server with SIGTERM. Maybe it hang, let's collect stacktraces.
echo -e "Possible deadlock on shutdown (see gdb.log)$FAIL" >> /test_output/test_results.tsv
# Add a special status just in case, so it will be possible to find in the CI DB
echo -e "Warning: server did not stop yet$OK" >> /test_output/test_results.tsv
kill -TERM "$(pidof gdb)" ||:
sleep 5
# The server could finally stop while we were terminating gdb, let's recheck if it's still running
kill -s 0 $pid || return
echo -e "Possible deadlock on shutdown (see gdb.log)$FAIL" >> /test_output/test_results.tsv
echo "thread apply all backtrace (on stop)" >> /test_output/gdb.log
timeout 30m gdb -batch -ex 'thread apply all backtrace' -p "$pid" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log
clickhouse stop --force

30
tests/clickhouse-test
View File

@ -278,7 +278,7 @@ def need_retry(args, stdout, stderr, total_time):
def get_processlist_with_stacktraces(args):
try:
if args.replicated_database:
return clickhouse_execute_json(
return clickhouse_execute(
args,
"""
SELECT materialize(hostName() || '::' || tcpPort()::String) as host_port, *
@ -295,14 +295,14 @@ def get_processlist_with_stacktraces(args):
WHERE query NOT LIKE '%system.processes%'
GROUP BY p.*
))
ORDER BY elapsed DESC
ORDER BY elapsed DESC FORMAT Vertical
""",
settings={
"allow_introspection_functions": 1,
},
)
else:
return clickhouse_execute_json(
return clickhouse_execute(
args,
"""
SELECT
@ -315,7 +315,7 @@ def get_processlist_with_stacktraces(args):
JOIN system.stack_trace s USING (query_id)
WHERE query NOT LIKE '%system.processes%'
GROUP BY p.*
ORDER BY elapsed DESC
ORDER BY elapsed DESC FORMAT Vertical
""",
settings={
"allow_introspection_functions": 1,
@ -582,6 +582,9 @@ class MergeTreeSettingsRandomizer:
"vertical_merge_algorithm_min_columns_to_activate": threshold_generator(
0.4, 0.4, 1, 100
),
"allow_vertical_merges_from_compact_to_wide_parts": lambda: random.randint(
0, 1
),
"min_merge_bytes_to_use_direct_io": threshold_generator(
0.25, 0.25, 1, 10 * 1024 * 1024 * 1024
),
@ -589,6 +592,10 @@ class MergeTreeSettingsRandomizer:
"merge_max_block_size": lambda: random.randint(1, 8192 * 3),
"index_granularity": lambda: random.randint(1, 65536),
"min_bytes_for_wide_part": threshold_generator(0.3, 0.3, 0, 1024 * 1024 * 1024),
"compress_marks": lambda: random.randint(0, 1),
"compress_primary_key": lambda: random.randint(0, 1),
"marks_compress_block_size": lambda: random.randint(8000, 100000),
"primary_key_compress_block_size": lambda: random.randint(8000, 100000),
}
@staticmethod
@ -2053,12 +2060,23 @@ def reportLogStats(args):
'No sharding key', 'No tables', 'Query: {}', 'Removed', 'Removed part {}', 'Removing parts.',
'Request URI: {}', 'Sending part {}', 'Sent handshake', 'Starting {}', 'Will mimic {}', 'Writing to {}',
'dropIfEmpty', 'loadAll {}', '{} ({}:{})', '{} -> {}', '{} {}', '{}: {}', 'Query was cancelled',
'Table {} already exists', '{}%', 'Cancelled merging parts', 'All replicas are lost',
'Table {} already exists.', '{}%', 'Cancelled merging parts', 'All replicas are lost',
'Cancelled mutating parts', 'Read object: {}', 'New segment: {}', 'Unknown geometry type {}',
'Table {} is not replicated', '{} {}.{} already exists', 'Attempt to read after eof',
'Replica {} already exists', 'Convert overflow', 'key must be a tuple', 'Division by zero',
'No part {} in committed state', 'Files set to {}', 'Bytes set to {}', 'Sharding key {} is not used',
'Cannot parse datetime', 'Bad get: has {}, requested {}', 'There is no {} in {}', 'Numeric overflow'
'Cannot parse datetime', 'Bad get: has {}, requested {}', 'There is no {} in {}', 'Numeric overflow',
'Polygon is not valid: {}', 'Decimal math overflow', '{} only accepts maps', 'Dictionary ({}) not found',
'Unknown format {}', 'Invalid IPv4 value', 'Invalid IPv6 value', 'Unknown setting {}',
'Unknown table function {}', 'Database {} already exists.', 'Table {} doesn''t exist',
'Invalid credentials', 'Part {} already exists', 'Invalid mode: {}', 'Log pulling is cancelled',
'JOIN {} cannot get JOIN keys', 'Unknown function {}{}', 'Cannot parse IPv6 {}',
'Not found address of host: {}', '{} must contain a tuple', 'Unknown codec family: {}',
'Expected const String column', 'Invalid partition format: {}', 'Cannot parse IPv4 {}',
'AST is too deep. Maximum: {}', 'Array sizes are too large: {}', 'Unable to connect to HDFS: {}',
'Shutdown is called for table', 'File is not inside {}',
'Table {} doesn''t exist', 'Database {} doesn''t exist', 'Table {}.{} doesn''t exist',
'File {} doesn''t exist', 'No such attribute ''{}''', 'User name ''{}'' is reserved'
) AS known_short_messages
SELECT count() AS c, message_format_string, substr(any(message), 1, 120)
FROM system.text_log

14
tests/integration/helpers/client.py
View File

@ -202,6 +202,16 @@ class CommandRequest:
self.timer = Timer(timeout, kill_process)
self.timer.start()
def remove_trash_from_stderr(self, stderr):
# FIXME https://github.com/ClickHouse/ClickHouse/issues/48181
if not stderr:
return stderr
lines = stderr.split("\n")
lines = [
x for x in lines if ("completion_queue" not in x and "Kick failed" not in x)
]
return "\n".join(lines)
def get_answer(self):
self.process.wait(timeout=DEFAULT_QUERY_TIMEOUT)
self.stdout_file.seek(0)
@ -218,7 +228,9 @@ class CommandRequest:
logging.debug(f"Timed out. Last stdout:{stdout}, stderr:{stderr}")
raise QueryTimeoutExceedException("Client timed out!")
if (self.process.returncode != 0 or stderr) and not self.ignore_error:
if (
self.process.returncode != 0 or self.remove_trash_from_stderr(stderr)
) and not self.ignore_error:
raise QueryRuntimeException(
"Client failed! Return code: {}, stderr: {}".format(
self.process.returncode, stderr

Some files were not shown because too many files have changed in this diff Show More