diff --git a/.exrc b/.exrc new file mode 100644 index 00000000000..162bd41ce4f --- /dev/null +++ b/.exrc @@ -0,0 +1 @@ +au BufRead,BufNewFile * set tabstop=4 softtabstop=0 expandtab shiftwidth=4 smarttab tags=tags,../tags diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 30a77a9b27f..c90df6e57b7 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -145,8 +145,8 @@ jobs: fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -190,8 +190,8 @@ jobs: fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -233,8 +233,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -276,8 +276,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -319,8 +319,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -364,8 +364,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -409,8 +409,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index da84500559a..f3d672136ef 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -209,8 +209,8 @@ jobs: fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -251,8 +251,8 @@ jobs: fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -295,8 +295,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -338,8 +338,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -381,8 +381,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -424,8 +424,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -467,8 +467,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -510,8 +510,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -556,8 +556,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -599,8 +599,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -644,8 +644,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -689,8 +689,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -734,8 +734,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -779,8 +779,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -824,8 +824,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -869,8 +869,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -914,8 +914,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -3011,6 +3011,150 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" + PerformanceComparisonAarch-0: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, func-tester-aarch64] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/performance_comparison + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Performance Comparison Aarch64 + REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Performance Comparison + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 performance_comparison_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + PerformanceComparisonAarch-1: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, func-tester-aarch64] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/performance_comparison + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Performance Comparison Aarch64 + REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Performance Comparison + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 performance_comparison_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + PerformanceComparisonAarch-2: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, func-tester-aarch64] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/performance_comparison + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Performance Comparison Aarch64 + REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Performance Comparison + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 performance_comparison_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + PerformanceComparisonAarch-3: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, func-tester-aarch64] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/performance_comparison + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Performance Comparison Aarch64 + REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse + RUN_BY_HASH_NUM=3 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Performance Comparison + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 performance_comparison_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" ############################################################################################## ###################################### SQLANCER FUZZERS ###################################### ############################################################################################## diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7dff1e205a1..9ebbe4e090d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -105,7 +105,7 @@ jobs: - name: Build run: | git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c1562d933a9..857e2c7f604 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -22,6 +22,8 @@ on: # yamllint disable-line rule:truthy jobs: CheckLabels: runs-on: [self-hosted, style-checker] + # Run the first check always, even if the CI is cancelled + if: ${{ always() }} steps: - name: Clear repository run: | @@ -112,7 +114,8 @@ jobs: StyleCheck: needs: DockerHubPush runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() || always() }} + # We need additional `&& ! cancelled()` to have the job being able to cancel + if: ${{ success() || failure() || ( always() && ! cancelled() ) }} steps: - name: Set envs run: | @@ -272,8 +275,8 @@ jobs: fetch-depth: 0 # for performance artifact - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -315,8 +318,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -360,8 +363,8 @@ jobs: fetch-depth: 0 # for performance artifact - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -403,8 +406,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -446,8 +449,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -489,8 +492,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -532,8 +535,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -575,8 +578,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -621,8 +624,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -664,8 +667,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -707,8 +710,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -750,8 +753,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -793,8 +796,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -836,8 +839,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -879,8 +882,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -922,8 +925,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -965,8 +968,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 8148905cec7..bf35ca76fc6 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -136,8 +136,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -178,8 +178,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -220,8 +220,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -263,8 +263,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -306,8 +306,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -349,8 +349,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -392,8 +392,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -437,8 +437,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -482,8 +482,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync --recursive - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --single-branch --depth=1 --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" diff --git a/.gitignore b/.gitignore index 09d3f4a4e33..6d94cade384 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ # logs *.log +*.debuglog *.stderr *.stdout diff --git a/.gitmodules b/.gitmodules index 293029ad171..ebeef312ae8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -65,12 +65,6 @@ [submodule "contrib/libgsasl"] path = contrib/libgsasl url = https://github.com/ClickHouse/libgsasl.git -[submodule "contrib/libcxx"] - path = contrib/libcxx - url = https://github.com/ClickHouse/libcxx.git -[submodule "contrib/libcxxabi"] - path = contrib/libcxxabi - url = https://github.com/ClickHouse/libcxxabi.git [submodule "contrib/snappy"] path = contrib/snappy url = https://github.com/ClickHouse/snappy.git @@ -290,3 +284,6 @@ [submodule "contrib/morton-nd"] path = contrib/morton-nd url = https://github.com/morton-nd/morton-nd +[submodule "contrib/xxHash"] + path = contrib/xxHash + url = https://github.com/Cyan4973/xxHash.git diff --git a/.vimrc b/.vimrc deleted file mode 100644 index ba996eb8a42..00000000000 --- a/.vimrc +++ /dev/null @@ -1,2 +0,0 @@ -au BufRead,BufNewFile ./* set tabstop=4 softtabstop=0 expandtab shiftwidth=4 smarttab tags=tags,../tags - diff --git a/README.md b/README.md index f90df9686c2..4f2483097d6 100644 --- a/README.md +++ b/README.md @@ -17,5 +17,7 @@ ClickHouse® is an open-source column-oriented database management system that a ## Upcoming events * [**v22.11 Release Webinar**](https://clickhouse.com/company/events/v22-11-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. -* [**ClickHouse Meetup at the Deutsche Bank office in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/289311596/) Hear from Deutsche Bank on why they chose ClickHouse for big sensitive data in a regulated environment. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management. -* [**AWS re:Invent**](https://clickhouse.com/company/events/aws-reinvent) Core members of the ClickHouse team -- including 2 of our founders -- will be at re:Invent from November 29 to December 3. We are available on the show floor, but are also determining interest in holding an event during the time there. +* [**ClickHosue Meetup at the RELEX Solutions office in Stockholm**](https://www.meetup.com/clickhouse-stockholm-user-group/events/289492084/) - Dec 1 - Formulate by RELEX is a Swedish promotion planning and analytics company. They will share why they chose ClickHouse for their real time analytics and forecasting solution. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management. +* [**ClickHouse Meetup at the Deutsche Bank office in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/289311596/) - Dec 5 - Hear from Deutsche Bank on why they chose ClickHouse for big sensitive data in a regulated environment. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management. +* [**ClickHouse Meetup at the Rokt offices in Manhattan**](https://www.meetup.com/clickhouse-new-york-user-group/events/289403909/) - Dec 6 - We are very excited to be holding our next in-person ClickHouse meetup at the Rokt offices in Manhattan. Featuring talks from Bloomberg, Disney Streaming, Prequel, Rokt, and ClickHouse + diff --git a/base/base/bit_cast.h b/base/base/bit_cast.h index b2b6915764d..8198991e98e 100644 --- a/base/base/bit_cast.h +++ b/base/base/bit_cast.h @@ -12,7 +12,21 @@ template std::decay_t bit_cast(const From & from) { + /** + * Assume the source value is 0xAABBCCDD (i.e. sizeof(from) == 4). + * Its BE representation is 0xAABBCCDD, the LE representation is 0xDDCCBBAA. + * Further assume, sizeof(res) == 8 and that res is initially zeroed out. + * With LE, the result after bit_cast will be 0xDDCCBBAA00000000 --> input value == output value. + * With BE, the result after bit_cast will be 0x00000000AABBCCDD --> input value == output value. + */ To res {}; - memcpy(static_cast(&res), &from, std::min(sizeof(res), sizeof(from))); + if constexpr (std::endian::native == std::endian::little) + memcpy(static_cast(&res), &from, std::min(sizeof(res), sizeof(from))); + else + { + uint32_t offset_to = (sizeof(res) > sizeof(from)) ? (sizeof(res) - sizeof(from)) : 0; + uint32_t offset_from = (sizeof(from) > sizeof(res)) ? (sizeof(from) - sizeof(res)) : 0; + memcpy(reinterpret_cast(&res) + offset_to, reinterpret_cast(&from) + offset_from, std::min(sizeof(res), sizeof(from))); + } return res; } diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 8ebd4ab55d3..ec7382846c2 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -167,7 +167,9 @@ add_contrib (c-ares-cmake c-ares) add_contrib (qpl-cmake qpl) add_contrib (morton-nd-cmake morton-nd) -add_contrib(annoy-cmake annoy) +add_contrib (annoy-cmake annoy) + +add_contrib (xxHash-cmake xxHash) # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear diff --git a/contrib/NuRaft b/contrib/NuRaft index e4e746a24eb..afc36dfa9b0 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit e4e746a24eb56861a86f3672771e3308d8c40722 +Subproject commit afc36dfa9b0beb45bc4cd935060631cc80ba04a5 diff --git a/contrib/libcxx b/contrib/libcxx deleted file mode 160000 index 4db7f838afd..00000000000 --- a/contrib/libcxx +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4db7f838afd3139eb3761694b04d31275df45d2d diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index 8dc154e9d91..21ed76f8b6f 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -1,6 +1,6 @@ include(CheckCXXCompilerFlag) -set(LIBCXX_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libcxx") +set(LIBCXX_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm-project/libcxx") set(SRCS "${LIBCXX_SOURCE_DIR}/src/algorithm.cpp" diff --git a/contrib/libcxxabi b/contrib/libcxxabi deleted file mode 160000 index a736a6b3c6a..00000000000 --- a/contrib/libcxxabi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a736a6b3c6a7b8aae2ebad629ca21b2c55b4820e diff --git a/contrib/libcxxabi-cmake/CMakeLists.txt b/contrib/libcxxabi-cmake/CMakeLists.txt index a59452eee9a..0473527912e 100644 --- a/contrib/libcxxabi-cmake/CMakeLists.txt +++ b/contrib/libcxxabi-cmake/CMakeLists.txt @@ -1,4 +1,4 @@ -set(LIBCXXABI_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libcxxabi") +set(LIBCXXABI_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm-project/libcxxabi") set(SRCS "${LIBCXXABI_SOURCE_DIR}/src/abort_message.cpp" diff --git a/contrib/llvm-project b/contrib/llvm-project index 3a39038345a..e61a81aa6fc 160000 --- a/contrib/llvm-project +++ b/contrib/llvm-project @@ -1 +1 @@ -Subproject commit 3a39038345a400e7e767811b142a94355d511215 +Subproject commit e61a81aa6fc529b469e2a54b7ce788606e138b5d diff --git a/contrib/poco b/contrib/poco index 76746b35d0e..79923422618 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 76746b35d0e254eaaba71dc3b79e46cba8cbb144 +Subproject commit 799234226187c0ae0b8c90f23465b25ed7956e56 diff --git a/contrib/xxHash b/contrib/xxHash new file mode 160000 index 00000000000..3078dc6039f --- /dev/null +++ b/contrib/xxHash @@ -0,0 +1 @@ +Subproject commit 3078dc6039f8c0bffcb1904f81cfe6b2c3209435 diff --git a/contrib/xxHash-cmake/CMakeLists.txt b/contrib/xxHash-cmake/CMakeLists.txt new file mode 100644 index 00000000000..314094e9523 --- /dev/null +++ b/contrib/xxHash-cmake/CMakeLists.txt @@ -0,0 +1,13 @@ +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/xxHash") +set (SRCS + "${LIBRARY_DIR}/xxhash.c" +) + +add_library(xxHash ${SRCS}) +target_include_directories(xxHash SYSTEM BEFORE INTERFACE "${LIBRARY_DIR}") + +# XXH_INLINE_ALL - Make all functions inline, with implementations being directly included within xxhash.h. Inlining functions is beneficial for speed on small keys. +# https://github.com/Cyan4973/xxHash/tree/v0.8.1#build-modifiers +target_compile_definitions(xxHash PUBLIC XXH_INLINE_ALL) + +add_library(ch_contrib::xxHash ALIAS xxHash) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 06c3c0d80f0..b3da09facda 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -6,29 +6,24 @@ FROM clickhouse/test-util:$FROM_TAG # Rust toolchain and libraries ENV RUSTUP_HOME=/rust/rustup ENV CARGO_HOME=/rust/cargo -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y -RUN chmod 777 -R /rust ENV PATH="/rust/cargo/env:${PATH}" ENV PATH="/rust/cargo/bin:${PATH}" -RUN rustup target add aarch64-unknown-linux-gnu && \ - rustup target add x86_64-apple-darwin && \ - rustup target add x86_64-unknown-freebsd && \ - rustup target add aarch64-apple-darwin && \ - rustup target add powerpc64le-unknown-linux-gnu -RUN apt-get install \ +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \ + chmod 777 -R /rust && \ + rustup target add aarch64-unknown-linux-gnu && \ + rustup target add x86_64-apple-darwin && \ + rustup target add x86_64-unknown-freebsd && \ + rustup target add aarch64-apple-darwin && \ + rustup target add powerpc64le-unknown-linux-gnu + +RUN apt-get update && \ + apt-get install --yes \ gcc-aarch64-linux-gnu \ build-essential \ libc6 \ libc6-dev \ - libc6-dev-arm64-cross \ - --yes - -# Install CMake 3.20+ for Rust compilation -# Used https://askubuntu.com/a/1157132 as reference -RUN apt purge cmake --yes -RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null -RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' -RUN apt update && apt install cmake --yes + libc6-dev-arm64-cross && \ + apt-get clean ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index de9125d565b..7359e0a9402 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -117,8 +117,7 @@ function clone_submodules contrib/cctz contrib/libcpuid contrib/double-conversion - contrib/libcxx - contrib/libcxxabi + contrib/llvm-project contrib/lz4 contrib/zstd contrib/fastops @@ -137,6 +136,7 @@ function clone_submodules contrib/hashidsxx contrib/c-ares contrib/morton-nd + contrib/xxHash ) git submodule sync diff --git a/docker/test/fuzzer/Dockerfile b/docker/test/fuzzer/Dockerfile index eb4b09c173f..aa71074c02a 100644 --- a/docker/test/fuzzer/Dockerfile +++ b/docker/test/fuzzer/Dockerfile @@ -38,7 +38,7 @@ COPY * / SHELL ["/bin/bash", "-c"] CMD set -o pipefail \ && cd /workspace \ - && /run-fuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log + && timeout -s 9 1h /run-fuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log # docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/fuzzer diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index dbb56b258ed..bd539ca978b 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -1,5 +1,5 @@ #!/bin/bash -# shellcheck disable=SC2086,SC2001,SC2046,SC2030,SC2031 +# shellcheck disable=SC2086,SC2001,SC2046,SC2030,SC2031,SC2010,SC2015 set -x @@ -10,11 +10,6 @@ set -e set -u set -o pipefail -trap "exit" INT TERM -# The watchdog is in the separate process group, so we have to kill it separately -# if the script terminates earlier. -trap 'kill $(jobs -pr) ${watchdog_pid:-} ||:' EXIT - stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "$script_dir" @@ -110,26 +105,6 @@ function configure EOL } -function watchdog -{ - sleep 1800 - - echo "Fuzzing run has timed out" - for _ in {1..10} - do - # Only kill by pid the particular client that runs the fuzzing, or else - # we can kill some clickhouse-client processes this script starts later, - # e.g. for checking server liveness. - if ! kill $fuzzer_pid - then - break - fi - sleep 1 - done - - kill -9 -- $fuzzer_pid ||: -} - function filter_exists_and_template { local path @@ -175,8 +150,6 @@ function fuzz mkdir -p /var/run/clickhouse-server - # interferes with gdb - export CLICKHOUSE_WATCHDOG_ENABLE=0 # NOTE: we use process substitution here to preserve keep $! as a pid of clickhouse-server clickhouse-server --config-file db/config.xml --pid-file /var/run/clickhouse-server/clickhouse-server.pid -- --path db 2>&1 | pigz > server.log.gz & server_pid=$! @@ -214,7 +187,7 @@ detach quit " > script.gdb - gdb -batch -command script.gdb -p $server_pid & + gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" & sleep 5 # gdb will send SIGSTOP, spend some time loading debug info and then send SIGCONT, wait for it (up to send_timeout, 300s) time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||: @@ -236,7 +209,7 @@ quit # SC2012: Use find instead of ls to better handle non-alphanumeric filenames. They are all alphanumeric. # SC2046: Quote this to prevent word splitting. Actually I need word splitting. # shellcheck disable=SC2012,SC2046 - clickhouse-client \ + timeout -s TERM --preserve-status 30m clickhouse-client \ --receive_timeout=10 \ --receive_data_timeout_ms=10000 \ --stacktrace \ @@ -249,16 +222,6 @@ quit fuzzer_pid=$! echo "Fuzzer pid is $fuzzer_pid" - # Start a watchdog that should kill the fuzzer on timeout. - # The shell won't kill the child sleep when we kill it, so we have to put it - # into a separate process group so that we can kill them all. - set -m - watchdog & - watchdog_pid=$! - set +m - # Check that the watchdog has started. - kill -0 $watchdog_pid - # Wait for the fuzzer to complete. # Note that the 'wait || ...' thing is required so that the script doesn't # exit because of 'set -e' when 'wait' returns nonzero code. @@ -266,8 +229,6 @@ quit wait "$fuzzer_pid" || fuzzer_exit_code=$? echo "Fuzzer exit code is $fuzzer_exit_code" - kill -- -$watchdog_pid ||: - # If the server dies, most often the fuzzer returns code 210: connetion # refused, and sometimes also code 32: attempt to read after eof. For # simplicity, check again whether the server is accepting connections, using @@ -333,6 +294,8 @@ quit pigz core.* mv core.*.gz core.gz fi + + dmesg -T | grep -q -F -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e 'oom-kill:constraint=CONSTRAINT_NONE' && echo "OOM in dmesg" ||: } case "$stage" in diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 7a034c741eb..cb23372d31f 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -295,6 +295,9 @@ if not args.use_existing_tables: reportStageEnd("create") +# Let's sync the data to avoid writeback affects performance +os.system("sync") + # By default, test all queries. queries_to_run = range(0, len(test_queries)) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index dd231f3ac66..5cb27d90b62 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -131,7 +131,14 @@ function stop() # Preserve the pid, since the server can hung after the PID will be deleted. pid="$(cat /var/run/clickhouse-server/clickhouse-server.pid)" - clickhouse stop --do-not-kill && return + # --max-tries is supported only since 22.12 + if dpkg --compare-versions "$(clickhouse local -q 'select version()')" ge "22.12"; then + # Increase default waiting timeout for sanitizers and debug builds + clickhouse stop --max-tries 180 --do-not-kill && return + else + clickhouse stop --do-not-kill && return + fi + # We failed to stop the server with SIGTERM. Maybe it hang, let's collect stacktraces. kill -TERM "$(pidof gdb)" ||: sleep 5 @@ -388,6 +395,8 @@ else rm -f /etc/clickhouse-server/config.d/storage_conf.xml ||: rm -f /etc/clickhouse-server/config.d/azure_storage_conf.xml ||: + # Turn on after 22.12 + rm -f /etc/clickhouse-server/config.d/compressed_marks_and_index.xml ||: # it uses recently introduced settings which previous versions may not have rm -f /etc/clickhouse-server/users.d/insert_keeper_retries.xml ||: @@ -456,7 +465,7 @@ else zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \ -e "Code: 236. DB::Exception: Cancelled mutating parts" \ -e "REPLICA_IS_ALREADY_ACTIVE" \ - -e "REPLICA_IS_ALREADY_EXIST" \ + -e "REPLICA_ALREADY_EXISTS" \ -e "ALL_REPLICAS_LOST" \ -e "DDLWorker: Cannot parse DDL task query" \ -e "RaftInstance: failed to accept a rpc connection due to error 125" \ @@ -487,6 +496,7 @@ else -e "Code: 269. DB::Exception: Destination table is myself" \ -e "Coordination::Exception: Connection loss" \ -e "MutateFromLogEntryTask" \ + -e "No connection to ZooKeeper, cannot get shared table ID" \ /var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "" > /test_output/bc_check_error_messages.txt \ && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index cb8c914e53d..e8c5e17024c 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ python3-pip \ shellcheck \ yamllint \ - && pip3 install black==22.8.0 boto3 codespell==2.2.1 dohq-artifactory PyGithub unidiff pylint==2.6.2 \ + && pip3 install black==22.8.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \ && apt-get clean \ && rm -rf /root/.cache/pip diff --git a/docker/test/style/process_style_check_result.py b/docker/test/style/process_style_check_result.py index 8c2110d64e5..6dc3d05d051 100755 --- a/docker/test/style/process_style_check_result.py +++ b/docker/test/style/process_style_check_result.py @@ -11,17 +11,19 @@ def process_result(result_folder): description = "" test_results = [] checks = ( - ("header duplicates", "duplicate_output.txt"), - ("shellcheck", "shellcheck_output.txt"), - ("style", "style_output.txt"), - ("black", "black_output.txt"), - ("typos", "typos_output.txt"), - ("whitespaces", "whitespaces_output.txt"), - ("workflows", "workflows_output.txt"), - ("doc typos", "doc_spell_output.txt"), + "duplicate includes", + "shellcheck", + "style", + "black", + "mypy", + "typos", + "whitespaces", + "workflows", + "docs spelling", ) - for name, out_file in checks: + for name in checks: + out_file = name.replace(" ", "_") + "_output.txt" full_path = os.path.join(result_folder, out_file) if not os.path.exists(full_path): logging.info("No %s check log on path %s", name, full_path) diff --git a/docker/test/style/run.sh b/docker/test/style/run.sh index 06ecadbfebf..80911bf8627 100755 --- a/docker/test/style/run.sh +++ b/docker/test/style/run.sh @@ -4,15 +4,17 @@ cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv echo "Check duplicates" | ts -./check-duplicate-includes.sh |& tee /test_output/duplicate_output.txt +./check-duplicate-includes.sh |& tee /test_output/duplicate_includes_output.txt echo "Check style" | ts ./check-style -n |& tee /test_output/style_output.txt echo "Check python formatting with black" | ts ./check-black -n |& tee /test_output/black_output.txt +echo "Check python type hinting with mypy" | ts +./check-mypy -n |& tee /test_output/mypy_output.txt echo "Check typos" | ts ./check-typos |& tee /test_output/typos_output.txt echo "Check docs spelling" | ts -./check-doc-aspell |& tee /test_output/doc_spell_output.txt +./check-doc-aspell |& tee /test_output/docs_spelling_output.txt echo "Check whitespaces" | ts ./check-whitespaces -n |& tee /test_output/whitespaces_output.txt echo "Check workflows" | ts diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index 57544bdc090..f1cf029e9a2 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -13,6 +13,7 @@ RUN apt-get update \ apt-transport-https \ apt-utils \ ca-certificates \ + curl \ dnsutils \ gnupg \ iputils-ping \ @@ -24,10 +25,16 @@ RUN apt-get update \ && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \ && apt-key add /tmp/llvm-snapshot.gpg.key \ && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ - && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \ + && echo "deb https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \ /etc/apt/sources.list \ && apt-get clean +# Install cmake 3.20+ for rust support +# Used https://askubuntu.com/a/1157132 as reference +RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | \ + gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg && \ + echo "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" >> /etc/apt/sources.list + # initial packages RUN apt-get update \ && apt-get install \ @@ -37,7 +44,6 @@ RUN apt-get update \ clang-${LLVM_VERSION} \ clang-tidy-${LLVM_VERSION} \ cmake \ - curl \ fakeroot \ gdb \ git \ diff --git a/docs/en/engines/table-engines/mergetree-family/annindexes.md b/docs/en/engines/table-engines/mergetree-family/annindexes.md index 3b2431e4b5b..e482926f400 100644 --- a/docs/en/engines/table-engines/mergetree-family/annindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/annindexes.md @@ -2,13 +2,20 @@ The main task that indexes achieve is to quickly find nearest neighbors for multidimensional data. An example of such a problem can be finding similar pictures (texts) for a given picture (text). That problem can be reduced to finding the nearest [embeddings](https://cloud.google.com/architecture/overview-extracting-and-serving-feature-embeddings-for-machine-learning). They can be created from data using [UDF](../../../sql-reference/functions/index.md#executable-user-defined-functions). -The next query finds the closest neighbors in N-dimensional space using the L2 (Euclidean) distance: +The next queries find the closest neighbors in N-dimensional space using the L2 (Euclidean) distance: ``` sql SELECT * FROM table_name WHERE L2Distance(Column, Point) < MaxDistance LIMIT N ``` + +``` sql +SELECT * +FROM table_name +ORDER BY L2Distance(Column, Point) +LIMIT N +``` But it will take some time for execution because of the long calculation of the distance between `TargetEmbedding` and all other vectors. This is where ANN indexes can help. They store a compact approximation of the search space (e.g. using clustering, search trees, etc.) and are able to compute approximate neighbors quickly. ## Indexes Structure @@ -34,26 +41,27 @@ Approximate Nearest Neighbor Search Indexes (`ANNIndexes`) are similar to skip i In these queries, `DistanceFunction` is selected from [distance functions](../../../sql-reference/functions/distance-functions). `Point` is a known vector (something like `(0.1, 0.1, ... )`). To avoid writing large vectors, use [client parameters](../../../interfaces/cli.md#queries-with-parameters-cli-queries-with-parameters). `Value` - a float value that will bound the neighbourhood. -!!! note "Note" - ANN index can't speed up query that satisfies both types(`where + order by`, only one of them). All queries must have the limit, as algorithms are used to find nearest neighbors and need a specific number of them. +:::note +ANN index can't speed up query that satisfies both types (`where + order by`, only one of them). All queries must have the limit, as algorithms are used to find nearest neighbors and need a specific number of them. +::: -!!! note "Note" - Indexes are applied only to queries with a limit less than the `max_limit_for_ann_queries` setting. This helps to avoid memory overflows in queries with a large limit. `max_limit_for_ann_queries` setting can be changed if you know you can provide enough memory. The default value is `1000000`. +:::note +Indexes are applied only to queries with a limit less than the `max_limit_for_ann_queries` setting. This helps to avoid memory overflows in queries with a large limit. `max_limit_for_ann_queries` setting can be changed if you know you can provide enough memory. The default value is `1000000`. +::: Both types of queries are handled the same way. The indexes get `n` neighbors (where `n` is taken from the `LIMIT` clause) and work with them. In `ORDER BY` query they remember the numbers of all parts of the granule that have at least one of neighbor. In `WHERE` query they remember only those parts that satisfy the requirements. - ## Create table with ANNIndex -This feature is disabled by default. To enable it, set `allow_experimental_annoy_index` to 1. Also, this feature is disabled for arm, due to likely problems with the algorithm. +This feature is disabled by default. To enable it, set `allow_experimental_annoy_index` to 1. Also, this feature is disabled on ARM, due to likely problems with the algorithm. ```sql CREATE TABLE t ( `id` Int64, - `number` Tuple(Float32, Float32, Float32), - INDEX x number TYPE annoy GRANULARITY N + `data` Tuple(Float32, Float32, Float32), + INDEX ann_index_name data TYPE ann_index_type(ann_index_parameters) GRANULARITY N ) ENGINE = MergeTree ORDER BY id; @@ -63,8 +71,8 @@ ORDER BY id; CREATE TABLE t ( `id` Int64, - `number` Array(Float32), - INDEX x number TYPE annoy GRANULARITY N + `data` Array(Float32), + INDEX ann_index_name data TYPE ann_index_type(ann_index_parameters) GRANULARITY N ) ENGINE = MergeTree ORDER BY id; @@ -73,7 +81,7 @@ ORDER BY id; With greater `GRANULARITY` indexes remember the data structure better. The `GRANULARITY` indicates how many granules will be used to construct the index. The more data is provided for the index, the more of it can be handled by one index and the more chances that with the right hyperparameters the index will remember the data structure better. But some indexes can't be built if they don't have enough data, so this granule will always participate in the query. For more information, see the description of indexes. As the indexes are built only during insertions into table, `INSERT` and `OPTIMIZE` queries are slower than for ordinary table. At this stage indexes remember all the information about the given data. ANNIndexes should be used if you have immutable or rarely changed data and many read requests. - + You can create your table with index which uses certain algorithm. Now only indices based on the following algorithms are supported: # Index list @@ -91,8 +99,8 @@ __Examples__: CREATE TABLE t ( id Int64, - number Tuple(Float32, Float32, Float32), - INDEX x number TYPE annoy(T) GRANULARITY N + data Tuple(Float32, Float32, Float32), + INDEX ann_index_name data TYPE annoy(NumTrees, DistanceName) GRANULARITY N ) ENGINE = MergeTree ORDER BY id; @@ -102,18 +110,30 @@ ORDER BY id; CREATE TABLE t ( id Int64, - number Array(Float32), - INDEX x number TYPE annoy(T) GRANULARITY N + data Array(Float32), + INDEX ann_index_name data TYPE annoy(NumTrees, DistanceName) GRANULARITY N ) ENGINE = MergeTree ORDER BY id; ``` -!!! note "Note" - Table with array field will work faster, but all arrays **must** have same length. Use [CONSTRAINT](../../../sql-reference/statements/create/table.md#constraints) to avoid errors. For example, `CONSTRAINT constraint_name_1 CHECK length(number) = 256`. -Parameter `T` is the number of trees which algorithm will create. The bigger it is, the slower (approximately linear) it works (in both `CREATE` and `SELECT` requests), but the better accuracy you get (adjusted for randomness). +:::note +Table with array field will work faster, but all arrays **must** have same length. Use [CONSTRAINT](../../../sql-reference/statements/create/table.md#constraints) to avoid errors. For example, `CONSTRAINT constraint_name_1 CHECK length(data) = 256`. +::: -Annoy supports only `L2Distance`. +Parameter `NumTrees` is the number of trees which the algorithm will create. The bigger it is, the slower (approximately linear) it works (in both `CREATE` and `SELECT` requests), but the better accuracy you get (adjusted for randomness). By default it is set to `100`. Parameter `DistanceName` is name of distance function. By default it is set to `L2Distance`. It can be set without changing first parameter, for example +```sql +CREATE TABLE t +( + id Int64, + data Array(Float32), + INDEX ann_index_name data TYPE annoy('cosineDistance') GRANULARITY N +) +ENGINE = MergeTree +ORDER BY id; +``` + +Annoy supports `L2Distance` and `cosineDistance`. In the `SELECT` in the settings (`ann_index_select_query_params`) you can specify the size of the internal buffer (more details in the description above or in the [original repository](https://github.com/spotify/annoy)). During the query it will inspect up to `search_k` nodes which defaults to `n_trees * n` if not provided. `search_k` gives you a run-time tradeoff between better accuracy and speed. diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index ead1a76992e..4867140789f 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -85,7 +85,7 @@ Example of setting the addresses of the auxiliary ZooKeeper cluster: ``` -To store table datameta in a auxiliary ZooKeeper cluster instead of default ZooKeeper cluster, we can use the SQL to create table with +To store table metadata in an auxiliary ZooKeeper cluster instead of default ZooKeeper cluster, we can use the SQL to create table with ReplicatedMergeTree engine as follow: ``` diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 3221b1a06fa..731348abfe7 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1456,6 +1456,10 @@ If setting [input_format_with_types_use_header](../operations/settings/settings. the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped. ::: +## RowBinary format settings {#row-binary-format-settings} + +- [format_binary_max_string_size](../operations/settings/settings.md#format_binary_max_string_size) - The maximum allowed size for String in RowBinary format. Default value - `1GiB`. + ## Values {#data-format-values} Prints every row in brackets. Rows are separated by commas. There is no comma after the last row. The values inside the brackets are also comma-separated. Numbers are output in a decimal format without quotes. Arrays are output in square brackets. Strings, dates, and dates with times are output in quotes. Escaping rules and parsing are similar to the [TabSeparated](#tabseparated) format. During formatting, extra spaces aren’t inserted, but during parsing, they are allowed and skipped (except for spaces inside array values, which are not allowed). [NULL](../sql-reference/syntax.md) is represented as `NULL`. diff --git a/docs/en/operations/caches.md b/docs/en/operations/caches.md index 3aeae7d1c9d..86760ec245f 100644 --- a/docs/en/operations/caches.md +++ b/docs/en/operations/caches.md @@ -11,6 +11,7 @@ Main cache types: - `mark_cache` — Cache of marks used by table engines of the [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) family. - `uncompressed_cache` — Cache of uncompressed data used by table engines of the [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) family. +- Operating system page cache (used indirectly, for files with actual data). Additional cache types: @@ -22,10 +23,4 @@ Additional cache types: - Schema inference cache. - [Filesystem cache](storing-data.md) over S3, Azure, Local and other disks. -Indirectly used: - -- OS page cache. - -To drop cache, use [SYSTEM DROP ... CACHE](../sql-reference/statements/system.md) statements. - -[Original article](https://clickhouse.com/docs/en/operations/caches/) +To drop one of the caches, use [SYSTEM DROP ... CACHE](../sql-reference/statements/system.md#drop-mark-cache) statements. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 7494f3db71a..2fc6e64b7eb 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -668,7 +668,7 @@ log_query_views=1 ## log_formatted_queries {#settings-log-formatted-queries} -Allows to log formatted queries to the [system.query_log](../../operations/system-tables/query_log.md) system table (populates `formatted_query` column in the [system.query_log](../../operations/system-tables/query_log.md)). +Allows to log formatted queries to the [system.query_log](../../operations/system-tables/query_log.md) system table (populates `formatted_query` column in the [system.query_log](../../operations/system-tables/query_log.md)). Possible values: @@ -1807,6 +1807,41 @@ See also: - System table [trace_log](../../operations/system-tables/trace_log.md/#system_tables-trace_log) +## memory_profiler_step {#memory_profiler_step} + +Sets the step of memory profiler. Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stacktrace and will write it into [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log). + +Possible values: + +- A positive integer number of bytes. + +- 0 for turning off the memory profiler. + +Default value: 4,194,304 bytes (4 MiB). + +## memory_profiler_sample_probability {#memory_profiler_sample_probability} + +Sets the probability of collecting stacktraces at random allocations and deallocations and writing them into [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log). + +Possible values: + +- A positive floating-point number in the range [0..1]. + +- 0.0 for turning off the memory sampling. + +Default value: 0.0. + +## trace_profile_events {#trace_profile_events} + +Enables or disables collecting stacktraces on each update of profile events along with the name of profile event and the value of increment and sending them into [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log). + +Possible values: + +- 1 — Tracing of profile events enabled. +- 0 — Tracing of profile events disabled. + +Default value: 0. + ## allow_introspection_functions {#settings-allow_introspection_functions} Enables or disables [introspections functions](../../sql-reference/functions/introspection.md) for query profiling. @@ -4829,3 +4864,11 @@ Disabled by default. Allow skipping columns with unsupported types while schema inference for format BSONEachRow. Disabled by default. + +## RowBinary format settings {#row-binary-format-settings} + +### format_binary_max_string_size {#format_binary_max_string_size} + +The maximum allowed size for String in RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit. + +Default value: `1GiB` diff --git a/docs/en/operations/system-tables/trace_log.md b/docs/en/operations/system-tables/trace_log.md index 0effe085b80..6299aafcae2 100644 --- a/docs/en/operations/system-tables/trace_log.md +++ b/docs/en/operations/system-tables/trace_log.md @@ -5,7 +5,8 @@ slug: /en/operations/system-tables/trace_log Contains stack traces collected by the sampling query profiler. -ClickHouse creates this table when the [trace_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) server configuration section is set. Also the [query_profiler_real_time_period_ns](../../operations/settings/settings.md#query_profiler_real_time_period_ns) and [query_profiler_cpu_time_period_ns](../../operations/settings/settings.md#query_profiler_cpu_time_period_ns) settings should be set. +ClickHouse creates this table when the [trace_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) server configuration section is set. Also see settings: [query_profiler_real_time_period_ns](../../operations/settings/settings.md#query_profiler_real_time_period_ns), [query_profiler_cpu_time_period_ns](../../operations/settings/settings.md#query_profiler_cpu_time_period_ns), [memory_profiler_step](../../operations/settings/settings.md#memory_profiler_step), +[memory_profiler_sample_probability](../../operations/settings/settings.md#memory_profiler_sample_probability), [trace_profile_events](../../operations/settings/settings.md#trace_profile_events). To analyze logs, use the `addressToLine`, `addressToLineWithInlines`, `addressToSymbol` and `demangle` introspection functions. @@ -29,6 +30,8 @@ Columns: - `CPU` represents collecting stack traces by CPU time. - `Memory` represents collecting allocations and deallocations when memory allocation exceeds the subsequent watermark. - `MemorySample` represents collecting random allocations and deallocations. + - `MemoryPeak` represents collecting updates of peak memory usage. + - `ProfileEvent` represents collecting of increments of profile events. - `thread_number` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Thread identifier. @@ -36,6 +39,12 @@ Columns: - `trace` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. +- `size` ([Int64](../../sql-reference/data-types/int-uint.md)) - For trace types `Memory`, `MemorySample` or `MemoryPeak` is the amount of memory allocated, for other trace types is 0. + +- `event` ([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md)) - For trace type `ProfileEvent` is the name of updated profile event, for other trace types is an empty string. + +- `increment` ([UInt64](../../sql-reference/data-types/int-uint.md)) - For trace type `ProfileEvent` is the amount of incremnt of profile event, for other trace types is 0. + **Example** ``` sql diff --git a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md index 34f875e2138..1e0b1d88c6e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md @@ -32,8 +32,8 @@ The null hypothesis is that means of populations are equal. Normal distribution - calculated t-statistic. [Float64](../../../sql-reference/data-types/float.md). - calculated p-value. [Float64](../../../sql-reference/data-types/float.md). -- [calculated confidence-interval-low.] [Float64](../../../sql-reference/data-types/float.md). -- [calculated confidence-interval-high.] [Float64](../../../sql-reference/data-types/float.md). +- calculated confidence-interval-low. [Float64](../../../sql-reference/data-types/float.md). +- calculated confidence-interval-high. [Float64](../../../sql-reference/data-types/float.md). **Example** diff --git a/docs/en/sql-reference/functions/arithmetic-functions.md b/docs/en/sql-reference/functions/arithmetic-functions.md index ece50591ef9..56f3a88b28b 100644 --- a/docs/en/sql-reference/functions/arithmetic-functions.md +++ b/docs/en/sql-reference/functions/arithmetic-functions.md @@ -161,3 +161,140 @@ Result: │ -1 │ └─────────────┘ ``` + +## multiplyDecimal(a, b[, result_scale]) + +Performs multiplication on two decimals. Result value will be of type [Decimal256](../../sql-reference/data-types/decimal.md). +Result scale can be explicitly specified by `result_scale` argument (const Integer in range `[0, 76]`). If not specified, the result scale is the max scale of given arguments. + +:::note +These functions work significantly slower than usual `multiply`. +In case you don't really need controlled precision and/or need fast computation, consider using [multiply](#multiply) +::: + +**Syntax** + +```sql +multiplyDecimal(a, b[, result_scale]) +``` + +**Arguments** + +- `a` — First value: [Decimal](../../sql-reference/data-types/decimal.md). +- `b` — Second value: [Decimal](../../sql-reference/data-types/decimal.md). +- `result_scale` — Scale of result: [Int/UInt](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- The result of multiplication with given scale. + +Type: [Decimal256](../../sql-reference/data-types/decimal.md). + +**Example** + +```text +┌─multiplyDecimal(toDecimal256(-12, 0), toDecimal32(-2.1, 1), 1)─┐ +│ 25.2 │ +└────────────────────────────────────────────────────────────────┘ +``` + +**Difference from regular multiplication:** +```sql +SELECT toDecimal64(-12.647, 3) * toDecimal32(2.1239, 4); +SELECT toDecimal64(-12.647, 3) as a, toDecimal32(2.1239, 4) as b, multiplyDecimal(a, b); +``` + +```text +┌─multiply(toDecimal64(-12.647, 3), toDecimal32(2.1239, 4))─┐ +│ -26.8609633 │ +└───────────────────────────────────────────────────────────┘ +┌─multiplyDecimal(toDecimal64(-12.647, 3), toDecimal32(2.1239, 4))─┐ +│ -26.8609 │ +└──────────────────────────────────────────────────────────────────┘ +``` + +```sql +SELECT + toDecimal64(-12.647987876, 9) AS a, + toDecimal64(123.967645643, 9) AS b, + multiplyDecimal(a, b); + +SELECT + toDecimal64(-12.647987876, 9) AS a, + toDecimal64(123.967645643, 9) AS b, + a * b; +``` + +```text +┌─────────────a─┬─────────────b─┬─multiplyDecimal(toDecimal64(-12.647987876, 9), toDecimal64(123.967645643, 9))─┐ +│ -12.647987876 │ 123.967645643 │ -1567.941279108 │ +└───────────────┴───────────────┴───────────────────────────────────────────────────────────────────────────────┘ + +Received exception from server (version 22.11.1): +Code: 407. DB::Exception: Received from localhost:9000. DB::Exception: Decimal math overflow: While processing toDecimal64(-12.647987876, 9) AS a, toDecimal64(123.967645643, 9) AS b, a * b. (DECIMAL_OVERFLOW) +``` + +## divideDecimal(a, b[, result_scale]) + +Performs division on two decimals. Result value will be of type [Decimal256](../../sql-reference/data-types/decimal.md). +Result scale can be explicitly specified by `result_scale` argument (const Integer in range `[0, 76]`). If not specified, the result scale is the max scale of given arguments. + +:::note +These function work significantly slower than usual `divide`. +In case you don't really need controlled precision and/or need fast computation, consider using [divide](#divide). +::: + +**Syntax** + +```sql +divideDecimal(a, b[, result_scale]) +``` + +**Arguments** + +- `a` — First value: [Decimal](../../sql-reference/data-types/decimal.md). +- `b` — Second value: [Decimal](../../sql-reference/data-types/decimal.md). +- `result_scale` — Scale of result: [Int/UInt](../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- The result of division with given scale. + +Type: [Decimal256](../../sql-reference/data-types/decimal.md). + +**Example** + +```text +┌─divideDecimal(toDecimal256(-12, 0), toDecimal32(2.1, 1), 10)─┐ +│ -5.7142857142 │ +└──────────────────────────────────────────────────────────────┘ +``` + +**Difference from regular division:** +```sql +SELECT toDecimal64(-12, 1) / toDecimal32(2.1, 1); +SELECT toDecimal64(-12, 1) as a, toDecimal32(2.1, 1) as b, divideDecimal(a, b, 1), divideDecimal(a, b, 5); +``` + +```text +┌─divide(toDecimal64(-12, 1), toDecimal32(2.1, 1))─┐ +│ -5.7 │ +└──────────────────────────────────────────────────┘ + +┌───a─┬───b─┬─divideDecimal(toDecimal64(-12, 1), toDecimal32(2.1, 1), 1)─┬─divideDecimal(toDecimal64(-12, 1), toDecimal32(2.1, 1), 5)─┐ +│ -12 │ 2.1 │ -5.7 │ -5.71428 │ +└─────┴─────┴────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +```sql +SELECT toDecimal64(-12, 0) / toDecimal32(2.1, 1); +SELECT toDecimal64(-12, 0) as a, toDecimal32(2.1, 1) as b, divideDecimal(a, b, 1), divideDecimal(a, b, 5); +``` + +```text +DB::Exception: Decimal result's scale is less than argument's one: While processing toDecimal64(-12, 0) / toDecimal32(2.1, 1). (ARGUMENT_OUT_OF_BOUND) + +┌───a─┬───b─┬─divideDecimal(toDecimal64(-12, 0), toDecimal32(2.1, 1), 1)─┬─divideDecimal(toDecimal64(-12, 0), toDecimal32(2.1, 1), 5)─┐ +│ -12 │ 2.1 │ -5.7 │ -5.71428 │ +└─────┴─────┴────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` diff --git a/docs/en/sql-reference/functions/distance-functions.md b/docs/en/sql-reference/functions/distance-functions.md index 88d6c2f3e17..293e02f8a54 100644 --- a/docs/en/sql-reference/functions/distance-functions.md +++ b/docs/en/sql-reference/functions/distance-functions.md @@ -474,13 +474,13 @@ Calculates the cosine distance between two vectors (the values of the tuples are **Syntax** ```sql -cosineDistance(tuple1, tuple2) +cosineDistance(vector1, vector2) ``` **Arguments** -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). +- `vector1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). +- `vector2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). **Returned value** @@ -488,7 +488,7 @@ cosineDistance(tuple1, tuple2) Type: [Float](../../sql-reference/data-types/float.md). -**Example** +**Examples** Query: diff --git a/docs/en/sql-reference/functions/ext-dict-functions.md b/docs/en/sql-reference/functions/ext-dict-functions.md index 1c33638da09..d9e811a5703 100644 --- a/docs/en/sql-reference/functions/ext-dict-functions.md +++ b/docs/en/sql-reference/functions/ext-dict-functions.md @@ -151,7 +151,7 @@ Perform the query: ``` sql SELECT - dictGet('ext-dict-mult', ('c1','c2'), number) AS val, + dictGet('ext-dict-mult', ('c1','c2'), number + 1) AS val, toTypeName(val) AS type FROM system.numbers LIMIT 3; diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index b9ec21bb59d..536249626e5 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -1865,6 +1865,17 @@ Next, specify the path to `libcatboostmodel.` in the clickhouse config ``` +For security and isolation reasons, the model evaluation does not run in the server process but in the clickhouse-library-bridge process. +At the first execution of `catboostEvaluate()`, the server starts the library bridge process if it is not running already. Both processes +communicate using a HTTP interface. By default, port `9012` is used. A different port can be specified as follows - this is useful if port +`9012` is already assigned to a different service. + +``` xml + + 9019 + +``` + 2. Train a catboost model using libcatboost See [Training and applying models](https://catboost.ai/docs/features/training.html#training) for how to train catboost models from a training data set. diff --git a/docs/en/sql-reference/statements/alter/projection.md b/docs/en/sql-reference/statements/alter/projection.md index 3f6f493aa89..908d28d7ab1 100644 --- a/docs/en/sql-reference/statements/alter/projection.md +++ b/docs/en/sql-reference/statements/alter/projection.md @@ -11,6 +11,14 @@ Projections store data in a format that optimizes query execution, this feature You can define one or more projections for a table, and during the query analysis the projection with the least data to scan will be selected by ClickHouse without modifying the query provided by the user. +:::note Disk usage + +Projections will create internally a new hidden table, this means that more IO and space on disk will be required. +Example, If the projection has defined a different primary key, all the data from the original table will be duplicated. +::: + +You can see more technical details about how projections work internally on this [page](/docs/en/guides/improving-query-performance/sparse-primary-indexes/sparse-primary-indexes-multiple.md/#option-3-projections). + ## Example filtering without using primary keys Creating the table: diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 6dbd6bf8136..68fb968c609 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -59,6 +59,28 @@ If the table already exists and `IF NOT EXISTS` is specified, the query won’t There can be other clauses after the `ENGINE` clause in the query. See detailed documentation on how to create tables in the descriptions of [table engines](../../../engines/table-engines/index.md#table_engines). +:::tip +In ClickHouse Cloud please split this into two steps: +1. Create the table structure + + ```sql + CREATE TABLE t1 + ENGINE = MergeTree + ORDER BY ... + # highlight-next-line + EMPTY AS + SELECT ... + ``` + +2. Populate the table + + ```sql + INSERT INTO t1 + SELECT ... + ``` + +::: + **Example** Query: @@ -159,7 +181,7 @@ ENGINE = engine PRIMARY KEY(expr1[, expr2,...]); ``` -:::warning +:::warning You can't combine both ways in one query. ::: @@ -215,7 +237,7 @@ ALTER TABLE codec_example MODIFY COLUMN float_value CODEC(Default); Codecs can be combined in a pipeline, for example, `CODEC(Delta, Default)`. -:::warning +:::warning You can’t decompress ClickHouse database files with external utilities like `lz4`. Instead, use the special [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) utility. ::: @@ -301,44 +323,44 @@ Encryption codecs: #### AES_128_GCM_SIV -`CODEC('AES-128-GCM-SIV')` — Encrypts data with AES-128 in [RFC 8452](https://tools.ietf.org/html/rfc8452) GCM-SIV mode. +`CODEC('AES-128-GCM-SIV')` — Encrypts data with AES-128 in [RFC 8452](https://tools.ietf.org/html/rfc8452) GCM-SIV mode. #### AES-256-GCM-SIV -`CODEC('AES-256-GCM-SIV')` — Encrypts data with AES-256 in GCM-SIV mode. +`CODEC('AES-256-GCM-SIV')` — Encrypts data with AES-256 in GCM-SIV mode. These codecs use a fixed nonce and encryption is therefore deterministic. This makes it compatible with deduplicating engines such as [ReplicatedMergeTree](../../../engines/table-engines/mergetree-family/replication.md) but has a weakness: when the same data block is encrypted twice, the resulting ciphertext will be exactly the same so an adversary who can read the disk can see this equivalence (although only the equivalence, without getting its content). -:::warning +:::warning Most engines including the "\*MergeTree" family create index files on disk without applying codecs. This means plaintext will appear on disk if an encrypted column is indexed. ::: -:::warning +:::warning If you perform a SELECT query mentioning a specific value in an encrypted column (such as in its WHERE clause), the value may appear in [system.query_log](../../../operations/system-tables/query_log.md). You may want to disable the logging. ::: **Example** ```sql -CREATE TABLE mytable +CREATE TABLE mytable ( x String Codec(AES_128_GCM_SIV) -) +) ENGINE = MergeTree ORDER BY x; ``` -:::note +:::note If compression needs to be applied, it must be explicitly specified. Otherwise, only encryption will be applied to data. ::: **Example** ```sql -CREATE TABLE mytable +CREATE TABLE mytable ( x String Codec(Delta, LZ4, AES_128_GCM_SIV) -) +) ENGINE = MergeTree ORDER BY x; ``` @@ -372,7 +394,7 @@ It’s possible to use tables with [ENGINE = Memory](../../../engines/table-engi 'REPLACE' query allows you to update the table atomically. -:::note +:::note This query is supported only for [Atomic](../../../engines/database-engines/atomic.md) database engine. ::: @@ -388,7 +410,7 @@ RENAME TABLE myNewTable TO myOldTable; Instead of above, you can use the following: ```sql -REPLACE TABLE myOldTable SELECT * FROM myOldTable WHERE CounterID <12345; +REPLACE TABLE myOldTable ENGINE = MergeTree() ORDER BY CounterID AS SELECT * FROM myOldTable WHERE CounterID <12345; ``` ### Syntax @@ -448,7 +470,7 @@ SELECT * FROM base.t1; You can add a comment to the table when you creating it. -:::note +:::note The comment is supported for all table engines except [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) and [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md). ::: diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index 5833c43f55d..85741117d2a 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -60,7 +60,7 @@ If you specify `POPULATE`, the existing table data is inserted into the view whe A `SELECT` query can contain `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`. Note that the corresponding conversions are performed independently on each block of inserted data. For example, if `GROUP BY` is set, data is aggregated during insertion, but only within a single packet of inserted data. The data won’t be further aggregated. The exception is when using an `ENGINE` that independently performs data aggregation, such as `SummingMergeTree`. -The execution of [ALTER](../../../sql-reference/statements/alter/view.md) queries on materialized views has limitations, so they might be inconvenient. If the materialized view uses the construction `TO [db.]name`, you can `DETACH` the view, run `ALTER` for the target table, and then `ATTACH` the previously detached (`DETACH`) view. +The execution of [ALTER](/docs/en/sql-reference/statements/alter/view.md) queries on materialized views has limitations, for example, you can not update the `SELECT` query, so this might be inconvenient. If the materialized view uses the construction `TO [db.]name`, you can `DETACH` the view, run `ALTER` for the target table, and then `ATTACH` the previously detached (`DETACH`) view. Note that materialized view is influenced by [optimize_on_insert](../../../operations/settings/settings.md#optimize-on-insert) setting. The data is merged before the insertion into a view. diff --git a/docs/en/sql-reference/statements/explain.md b/docs/en/sql-reference/statements/explain.md index c5995e067d1..5649486905e 100644 --- a/docs/en/sql-reference/statements/explain.md +++ b/docs/en/sql-reference/statements/explain.md @@ -10,7 +10,7 @@ Shows the execution plan of a statement. Syntax: ```sql -EXPLAIN [AST | SYNTAX | PLAN | PIPELINE | ESTIMATE | TABLE OVERRIDE] [setting = value, ...] +EXPLAIN [AST | SYNTAX | QUERY TREE | PLAN | PIPELINE | ESTIMATE | TABLE OVERRIDE] [setting = value, ...] [ SELECT ... | tableFunction(...) [COLUMNS (...)] [ORDER BY ...] [PARTITION BY ...] [PRIMARY KEY] [SAMPLE BY ...] [TTL ...] diff --git a/docs/ru/operations/external-authenticators/kerberos.md b/docs/ru/operations/external-authenticators/kerberos.md index 7b0702b2132..865ea639c89 100644 --- a/docs/ru/operations/external-authenticators/kerberos.md +++ b/docs/ru/operations/external-authenticators/kerberos.md @@ -98,7 +98,7 @@ ClickHouse предоставляет возможность аутентифи :::danger "Важно" - Если пользователь настроен для Kerberos-аутентификации, другие виды уатентификации будут для него недоступны. Если наряду с `kerberos` в определении пользователя будет указан какой-либо другой способ аутентификации, ClickHouse завершит работу. + Если пользователь настроен для Kerberos-аутентификации, другие виды аутентификации будут для него недоступны. Если наряду с `kerberos` в определении пользователя будет указан какой-либо другой способ аутентификации, ClickHouse завершит работу. :::info "" Ещё раз отметим, что кроме `users.xml`, необходимо также включить Kerberos в `config.xml`. diff --git a/docs/ru/sql-reference/functions/arithmetic-functions.md b/docs/ru/sql-reference/functions/arithmetic-functions.md index bc1d0a55128..4e040edcc70 100644 --- a/docs/ru/sql-reference/functions/arithmetic-functions.md +++ b/docs/ru/sql-reference/functions/arithmetic-functions.md @@ -159,3 +159,150 @@ SELECT min2(-1, 2); └─────────────┘ ``` +## multiplyDecimal(a, b[, result_scale]) + +Совершает умножение двух Decimal. Результат будет иметь тип [Decimal256](../../sql-reference/data-types/decimal.md). +Scale (размер дробной части) результат можно явно задать аргументом `result_scale` (целочисленная константа из интервала `[0, 76]`). +Если этот аргумент не задан, то scale результата будет равен наибольшему из scale обоих аргументов. + +**Синтаксис** + +```sql +multiplyDecimal(a, b[, result_scale]) +``` + +:::note +Эта функция работают гораздо медленнее обычной `multiply`. +В случае, если нет необходимости иметь фиксированную точность и/или нужны быстрые вычисления, следует использовать [multiply](#multiply). +::: + +**Аргументы** + +- `a` — Первый сомножитель/делимое: [Decimal](../../sql-reference/data-types/decimal.md). +- `b` — Второй сомножитель/делитель: [Decimal](../../sql-reference/data-types/decimal.md). +- `result_scale` — Scale результата: [Int/UInt](../../sql-reference/data-types/int-uint.md). + +**Возвращаемое значение** + +- Результат умножения с заданным scale. + +Тип: [Decimal256](../../sql-reference/data-types/decimal.md). + +**Примеры** + +```sql +SELECT multiplyDecimal(toDecimal256(-12, 0), toDecimal32(-2.1, 1), 1); +``` + +```text +┌─multiplyDecimal(toDecimal256(-12, 0), toDecimal32(-2.1, 1), 1)─┐ +│ 25.2 │ +└────────────────────────────────────────────────────────────────┘ +``` + +**Отличие от стандартных функций** +```sql +SELECT toDecimal64(-12.647, 3) * toDecimal32(2.1239, 4); +SELECT toDecimal64(-12.647, 3) as a, toDecimal32(2.1239, 4) as b, multiplyDecimal(a, b); +``` + +```text +┌─multiply(toDecimal64(-12.647, 3), toDecimal32(2.1239, 4))─┐ +│ -26.8609633 │ +└───────────────────────────────────────────────────────────┘ +┌─multiplyDecimal(toDecimal64(-12.647, 3), toDecimal32(2.1239, 4))─┐ +│ -26.8609 │ +└──────────────────────────────────────────────────────────────────┘ +``` + +```sql +SELECT + toDecimal64(-12.647987876, 9) AS a, + toDecimal64(123.967645643, 9) AS b, + multiplyDecimal(a, b); + +SELECT + toDecimal64(-12.647987876, 9) AS a, + toDecimal64(123.967645643, 9) AS b, + a * b; +``` + +```text +┌─────────────a─┬─────────────b─┬─multiplyDecimal(toDecimal64(-12.647987876, 9), toDecimal64(123.967645643, 9))─┐ +│ -12.647987876 │ 123.967645643 │ -1567.941279108 │ +└───────────────┴───────────────┴───────────────────────────────────────────────────────────────────────────────┘ + +Received exception from server (version 22.11.1): +Code: 407. DB::Exception: Received from localhost:9000. DB::Exception: Decimal math overflow: While processing toDecimal64(-12.647987876, 9) AS a, toDecimal64(123.967645643, 9) AS b, a * b. (DECIMAL_OVERFLOW) +``` + +## divideDecimal(a, b[, result_scale]) + +Совершает деление двух Decimal. Результат будет иметь тип [Decimal256](../../sql-reference/data-types/decimal.md). +Scale (размер дробной части) результат можно явно задать аргументом `result_scale` (целочисленная константа из интервала `[0, 76]`). +Если этот аргумент не задан, то scale результата будет равен наибольшему из scale обоих аргументов. + +**Синтаксис** + +```sql +divideDecimal(a, b[, result_scale]) +``` + +:::note +Эта функция работает гораздо медленнее обычной `divide`. +В случае, если нет необходимости иметь фиксированную точность и/или нужны быстрые вычисления, следует использовать [divide](#divide). +::: + +**Аргументы** + +- `a` — Первый сомножитель/делимое: [Decimal](../../sql-reference/data-types/decimal.md). +- `b` — Второй сомножитель/делитель: [Decimal](../../sql-reference/data-types/decimal.md). +- `result_scale` — Scale результата: [Int/UInt](../../sql-reference/data-types/int-uint.md). + +**Возвращаемое значение** + +- Результат деления с заданным scale. + +Тип: [Decimal256](../../sql-reference/data-types/decimal.md). + +**Примеры** + +```sql +SELECT divideDecimal(toDecimal256(-12, 0), toDecimal32(2.1, 1), 10); +``` + +```text +┌─divideDecimal(toDecimal256(-12, 0), toDecimal32(2.1, 1), 10)─┐ +│ -5.7142857142 │ +└──────────────────────────────────────────────────────────────┘ +``` + +**Отличие от стандартных функций** +```sql +SELECT toDecimal64(-12, 1) / toDecimal32(2.1, 1); +SELECT toDecimal64(-12, 1) as a, toDecimal32(2.1, 1) as b, divideDecimal(a, b, 1), divideDecimal(a, b, 5); +``` + +```text +┌─divide(toDecimal64(-12, 1), toDecimal32(2.1, 1))─┐ +│ -5.7 │ +└──────────────────────────────────────────────────┘ + +┌───a─┬───b─┬─divideDecimal(toDecimal64(-12, 1), toDecimal32(2.1, 1), 1)─┬─divideDecimal(toDecimal64(-12, 1), toDecimal32(2.1, 1), 5)─┐ +│ -12 │ 2.1 │ -5.7 │ -5.71428 │ +└─────┴─────┴────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + +```sql +SELECT toDecimal64(-12, 0) / toDecimal32(2.1, 1); +SELECT toDecimal64(-12, 0) as a, toDecimal32(2.1, 1) as b, divideDecimal(a, b, 1), divideDecimal(a, b, 5); +``` + +```text +DB::Exception: Decimal result's scale is less than argument's one: While processing toDecimal64(-12, 0) / toDecimal32(2.1, 1). (ARGUMENT_OUT_OF_BOUND) + +┌───a─┬───b─┬─divideDecimal(toDecimal64(-12, 0), toDecimal32(2.1, 1), 1)─┬─divideDecimal(toDecimal64(-12, 0), toDecimal32(2.1, 1), 5)─┐ +│ -12 │ 2.1 │ -5.7 │ -5.71428 │ +└─────┴─────┴────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────┘ +``` + diff --git a/docs/ru/sql-reference/functions/ext-dict-functions.md b/docs/ru/sql-reference/functions/ext-dict-functions.md index 9651ad52a76..e6cb878d1c7 100644 --- a/docs/ru/sql-reference/functions/ext-dict-functions.md +++ b/docs/ru/sql-reference/functions/ext-dict-functions.md @@ -151,7 +151,7 @@ LIMIT 3; ``` sql SELECT - dictGet('ext-dict-mult', ('c1','c2'), number) AS val, + dictGet('ext-dict-mult', ('c1','c2'), number + 1) AS val, toTypeName(val) AS type FROM system.numbers LIMIT 3; diff --git a/docs/tools/release.sh b/docs/tools/release.sh index 1d344457bf1..67499631baa 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -19,7 +19,7 @@ then # Will make a repository with website content as the only commit. git init git remote add origin "${GIT_PROD_URI}" - git config user.email "robot-clickhouse@clickhouse.com" + git config user.email "robot-clickhouse@users.noreply.github.com" git config user.name "robot-clickhouse" # Add files. diff --git a/docs/zh/engines/table-engines/integrations/kafka.md b/docs/zh/engines/table-engines/integrations/kafka.md index 707ee962ace..c6f11d9efce 100644 --- a/docs/zh/engines/table-engines/integrations/kafka.md +++ b/docs/zh/engines/table-engines/integrations/kafka.md @@ -74,7 +74,7 @@ Kafka 特性： 消费的消息会被自动追踪，因此每个消息在不同的消费组里只会记录一次。如果希望获得两次数据，则使用另一个组名创建副本。 -消费组可以灵活配置并且在集群之间同步。例如，如果群集中有10个主题和5个表副本，则每个副本将获得2个主题。 如果副本数量发生变化，主题将自动在副本中重新分配。了解更多信息请访问 http://kafka.apache.org/intro。 +消费组可以灵活配置并且在集群之间同步。例如，如果群集中有10个主题和5个表副本，则每个副本将获得2个主题。 如果副本数量发生变化，主题将自动在副本中重新分配。了解更多信息请访问 [http://kafka.apache.org/intro](http://kafka.apache.org/intro)。 `SELECT` 查询对于读取消息并不是很有用（调试除外），因为每条消息只能被读取一次。使用物化视图创建实时线程更实用。您可以这样做： diff --git a/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md b/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md index 13b4c368a96..e773a02fbc3 100644 --- a/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md +++ b/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md @@ -164,7 +164,7 @@ SETTINGS index_granularity = 8192, index_granularity_bytes = 0;
  • index_granularity: 显式设置为其默认值8192。这意味着对于每一组8192行，主索引将有一个索引条目，例如，如果表包含16384行，那么索引将有两个索引条目。

    • -
  • index_granularity_bytes: 设置为0表示禁止字适应索引粒度。自适应索引粒度意味着ClickHouse自动为一组n行创建一个索引条目 +
  • index_granularity_bytes: 设置为0表示禁止自适应索引粒度。自适应索引粒度意味着ClickHouse自动为一组n行创建一个索引条目
    • 如果n小于8192，但n行的合并行数据大小大于或等于10MB (index_granularity_bytes的默认值)或
    • n达到8192
      • @@ -777,7 +777,7 @@ ClickHouse现在创建了一个额外的索引来存储—每组4个连续的颗 如果我们想显著加快我们的两个示例查询——一个过滤具有特定UserID的行，一个过滤具有特定URL的行——那么我们需要使用多个主索引，通过使用这三个方法中的一个： - 新建一个不同主键的新表。 -- 创建一个雾化视图。 +- 创建一个物化视图。 - 增加projection。 这三个方法都会有效地将示例数据复制到另一个表中，以便重新组织表的主索引和行排序顺序。 @@ -992,7 +992,7 @@ Ok. :::note - 我们在视图的主键中切换键列的顺序(与原始表相比) -- 雾化视图由一个隐藏表支持，该表的行顺序和主索引基于给定的主键定义 +- 物化视图由一个隐藏表支持，该表的行顺序和主索引基于给定的主键定义 - 我们使用POPULATE关键字，以便用源表hits_UserID_URL中的所有887万行立即导入新的物化视图 - 如果在源表hits_UserID_URL中插入了新行，那么这些行也会自动插入到隐藏表中 - 实际上，隐式创建的隐藏表的行顺序和主索引与我们上面显式创建的辅助表相同: @@ -1082,7 +1082,7 @@ ALTER TABLE hits_UserID_URL ); ``` -雾化projection： +物化projection： ```sql ALTER TABLE hits_UserID_URL MATERIALIZE PROJECTION prj_url_userid; diff --git a/docs/zh/sql-reference/statements/select/limit-by.md b/docs/zh/sql-reference/statements/select/limit-by.md index 22052a4f814..50e3505b7fb 100644 --- a/docs/zh/sql-reference/statements/select/limit-by.md +++ b/docs/zh/sql-reference/statements/select/limit-by.md @@ -5,17 +5,17 @@ sidebar_label: LIMIT BY # LIMIT BY子句 {#limit-by-clause} -与查询 `LIMIT n BY expressions` 子句选择第一个 `n` 每个不同值的行 `expressions`. `LIMIT BY` 可以包含任意数量的 [表达式](../../../sql-reference/syntax.md#syntax-expressions). +一个使用`LIMIT n BY expressions`从句的查询会以去重后的`expressions`结果分组，每一分组选择前`n`行。`LIMIT BY`指定的值可以是任意数量的[表达式](../../../sql-reference/syntax.md#syntax-expressions)。 ClickHouse支持以下语法变体: - `LIMIT [offset_value, ]n BY expressions` - `LIMIT n OFFSET offset_value BY expressions` -在进行查询处理时，ClickHouse选择按排序键排序的数据。排序键设置显式地使用一个[ORDER BY](order-by.md#select-order-by)条款或隐式属性表的引擎(行顺序只是保证在使用[ORDER BY](order-by.md#select-order-by),否则不会命令行块由于多线程)。然后ClickHouse应用`LIMIT n BY 表达式`，并为每个不同的`表达式`组合返回前n行。如果指定了`OFFSET`，那么对于每个属于不同`表达式`组合的数据块，ClickHouse将跳过`offset_value`从块开始的行数，并最终返回最多`n`行的结果。如果`offset_value`大于数据块中的行数，则ClickHouse从数据块中返回零行。 +处理查询时，ClickHouse首先选择经由排序键排序过后的数据。排序键可以显式地使用[ORDER BY](order-by.md#select-order-by)从句指定，或隐式地使用表引擎使用的排序键（数据的顺序仅在使用[ORDER BY](order-by.md#select-order-by)时才可以保证，否则由于多线程处理，数据顺序会随机化）。然后ClickHouse执行`LIMIT n BY expressions`从句，将每一行按 `expressions` 的值进行分组，并对每一分组返回前`n`行。如果指定了`OFFSET`，那么对于每一分组，ClickHouse会跳过前`offset_value`行，接着返回前`n`行。如果`offset_value`大于某一分组的行数，ClickHouse会从分组返回0行。 !!! note "注" - `LIMIT BY` 是不相关的 [LIMIT](../../../sql-reference/statements/select/limit.md). 它们都可以在同一个查询中使用。 + `LIMIT BY`与[LIMIT](../../../sql-reference/statements/select/limit.md)没有关系。它们可以在同一个查询中使用。 ## 例 {#examples} @@ -53,9 +53,9 @@ SELECT * FROM limit_by ORDER BY id, val LIMIT 1, 2 BY id └────┴─────┘ ``` -该 `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` 查询返回相同的结果。 +与 `SELECT * FROM limit_by ORDER BY id, val LIMIT 2 OFFSET 1 BY id` 返回相同的结果。 -以下查询返回每个引用的前5个引用 `domain, device_type` 最多可与100行配对 (`LIMIT n BY + LIMIT`). +以下查询返回每个`domain,device_type`组合的前5个refferrer，总计返回至多100行(`LIMIT n BY + LIMIT`)。 ``` sql SELECT diff --git a/docs/zh/whats-new/security-changelog.md b/docs/zh/whats-new/security-changelog.md index a4e82241cb1..1e94e43fd83 100644 --- a/docs/zh/whats-new/security-changelog.md +++ b/docs/zh/whats-new/security-changelog.md @@ -3,6 +3,66 @@ slug: /zh/whats-new/security-changelog sidebar_position: 76 sidebar_label: 安全更新日志 --- +# 安全更新日志 +## 修复于ClickHouse 22.9.1.2603, 2022-09-22 +### CVE-2022-44011 +ClickHouse server中发现了一个堆缓冲区溢出问题。拥有向ClickHouse Server导入数据能力的恶意用户，可通过插入畸形CapnProto对象使ClickHouse Server对象崩溃。 + +修复已推送至版本22.9.1.2603, 22.8.2.11，22.7.4.16，22.6.6.16，22.3.12.19 + +作者：Kiojj(独立研究者) + +### CVE-2022-44010 +ClickHouse server中发现了一个堆缓冲区溢出问题。攻击者可发送一个特殊的HTTP请求至HTTP端口（默认监听在8123端口），该攻击可造成堆缓冲区溢出进而使ClickHouse server进程崩溃。执行该攻击无需认证。 + +修复版本已推送至版本22.9.1.2603，22.8.2.11，22.7.4.16，22.6.6.16，22.3.12.19 + +作者：Kiojj(独立研究者) + +## 修复于ClickHouse 21.10.2.15，2021-10-18 +### CVE-2021-43304 +在对恶意查询做语法分析时，ClickHouse的LZ4压缩编码会堆缓冲区溢出。LZ4:decompressImpl循环尤其是wildCopy(op, ip, copy_end)中的随意复制操作没有验证是否会导致超出目标缓冲区限制。 + +作者：JFrog 安全研究团队 + +### CVE-2021-43305 +在对恶意查询做语法分析时，ClickHouse的LZ4压缩编码会堆缓冲区溢出。LZ4:decompressImpl循环尤其是wildCopy(op, ip, copy_end)中的随意复制操作没有验证是否会导致超出目标缓冲区限制。 +该问题于CVE-2021-43304非常相似，但是无保护的copy操作存在于不同的wildCopy调用里。 + +作者：JFrog 安全研究团队 + +### CVE-2021-42387 +在对恶意查询做语法分析时，ClickHouse的LZ4:decompressImpl循环会从压缩数据中读取一个用户提供的16bit无符号值（'offset'）。这个offset后面在复制操作作为长度使用时，没有检查是否超过复制源的上限。 + +作者：JFrog 安全研究团队 + +### CVE-2021-42388 +在对恶意查询做语法分析时，ClickHouse的LZ4:decompressImpl循环会从压缩数据中读取一个用户提供的16bit无符号值（'offset'）。这个offset后面在复制操作作为长度使用时，没有检查是否越过复制源的下限。 + +作者：JFrog 安全研究团队 + +### CVE-2021-42389 +在对恶意查询做语法分析时，ClickHouse的Delta压缩编码存在除零错误。压缩缓存的首字节在取模时没有判断是否为0。 + +作者：JFrog 安全研究团队 + +### CVE-2021-42390 +在对恶意查询做语法分析时，ClickHouse的DeltaDouble压缩编码存在除零错误。压缩缓存的首字节在取模时没有判断是否为0。 + +作者：JFrog 安全研究团队 + +### CVE-2021-42391 +在对恶意查询做语法分析时， ClickHouse的Gorilla压缩编码存在除零错误，压缩缓存的首字节取模时没有判断是否为0。 + +作者：JFrog 安全研究团队 + +## 修复于ClickHouse 21.4.3.21，2021-04-12 +### CVE-2021-25263 +拥有CREATE DICTIONARY权限的攻击者，可以读取许可目录之外的任意文件。 + +修复已推送至版本20.8.18.32-lts，21.1.9.41-stable，21.2.9.41-stable，21.3.6.55-lts，21.4.3.21-stable以及更早期的版本。 + +作者：[Vyacheslav Egoshin](https://twitter.com/vegoshin) ## 修复于ClickHouse Release 19.14.3.3, 2019-09-10 {#fixed-in-clickhouse-release-19-14-3-3-2019-09-10} diff --git a/programs/client/clickhouse-client.xml b/programs/client/clickhouse-client.xml index 00f5b26eddf..2923de44045 100644 --- a/programs/client/clickhouse-client.xml +++ b/programs/client/clickhouse-client.xml @@ -15,18 +15,26 @@ {display_name} :) - {display_name} \x01\e[1;32m\x02:)\x01\e[0m\x02 - {display_name} \x01\e[1;31m\x02:)\x01\e[0m\x02 + {display_name} \e[1;32m:)\e[0m + {display_name} \e[1;31m:)\e[0m A, i.e. dependencies_info[B].dependent_database_objects contains A - /// and dependencies_info[A].dependencies contain B. - /// We need inverted graph to effectively maintain it on DDL queries that can modify the graph. - DependenciesInfos dependencies_info; }; /// Loads tables (and dictionaries) from specified databases @@ -92,25 +67,18 @@ private: Strings databases_to_load; ParsedTablesMetadata metadata; + TablesDependencyGraph referential_dependencies; + TablesDependencyGraph loading_dependencies; Poco::Logger * log; std::atomic tables_processed{0}; AtomicStopwatch stopwatch; ThreadPool pool; - void removeUnresolvableDependencies(bool remove_loaded); - + void buildDependencyGraph(); + void removeUnresolvableDependencies(); void loadTablesInTopologicalOrder(ThreadPool & pool); - - DependenciesInfosIter removeResolvedDependency(const DependenciesInfosIter & info_it, TableNames & independent_database_objects); - - void startLoadingIndependentTables(ThreadPool & pool, size_t level, ContextMutablePtr load_context); - - void checkCyclicDependencies() const; - - size_t getNumberOfTablesWithDependencies() const; - - void logDependencyGraph() const; + void startLoadingTables(ThreadPool & pool, ContextMutablePtr load_context, const std::vector & tables_to_load, size_t level); }; } diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp index e60fea46ed4..a409ddde9ec 100644 --- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp @@ -133,14 +133,25 @@ void AsynchronousReadIndirectBufferFromRemoteFS::prefetch() void AsynchronousReadIndirectBufferFromRemoteFS::setReadUntilPosition(size_t position) { - if (prefetch_future.valid()) + /// Do not reinitialize internal state in case the new end of range is already included. + /// Actually it is likely that we will anyway reinitialize it as seek method is called after + /// changing end position, but seek avoiding feature might help to avoid reinitialization, + /// so this check is useful to save the prefetch for the time when we try to avoid seek by + /// reading and ignoring some data. + if (!read_until_position || position > *read_until_position) { - prefetch_future.wait(); - prefetch_future = {}; - } + /// We must wait on future and reset the prefetch here, because otherwise there might be + /// a race between reading the data in the threadpool and impl->setReadUntilPosition() + /// which reinitializes internal remote read buffer (because if we have a new read range + /// then we need a new range request) and in case of reading from cache we need to request + /// and hold more file segment ranges from cache. + if (prefetch_future.valid()) + { + ProfileEvents::increment(ProfileEvents::RemoteFSCancelledPrefetches); + prefetch_future.wait(); + prefetch_future = {}; + } - if (position > read_until_position) - { read_until_position = position; impl->setReadUntilPosition(*read_until_position); } @@ -149,12 +160,6 @@ void AsynchronousReadIndirectBufferFromRemoteFS::setReadUntilPosition(size_t pos void AsynchronousReadIndirectBufferFromRemoteFS::setReadUntilEnd() { - if (prefetch_future.valid()) - { - prefetch_future.wait(); - prefetch_future = {}; - } - read_until_position = impl->getFileSize(); impl->setReadUntilPosition(*read_until_position); } diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 996268079e8..ed7b8182622 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -480,7 +480,8 @@ void S3ObjectStorage::copyObjectImpl( auto outcome = client_ptr->CopyObject(request); - if (!outcome.IsSuccess() && outcome.GetError().GetExceptionName() == "EntityTooLarge") + if (!outcome.IsSuccess() && (outcome.GetError().GetExceptionName() == "EntityTooLarge" + || outcome.GetError().GetExceptionName() == "InvalidRequest")) { // Can't come here with MinIO, MinIO allows single part upload for large objects. copyObjectMultipartImpl(src_bucket, src_key, dst_bucket, dst_key, head, metadata); return; diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 9c54a3526db..fe84d780714 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -180,6 +180,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.try_infer_datetimes = settings.input_format_try_infer_datetimes; format_settings.bson.output_string_as_string = settings.output_format_bson_string_as_string; format_settings.bson.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_bson_skip_fields_with_unsupported_types_in_schema_inference; + format_settings.max_binary_string_size = settings.format_binary_max_string_size; /// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context if (format_settings.schema.is_server) @@ -588,6 +589,19 @@ void FormatFactory::markFormatSupportsSubsetOfColumns(const String & name) target = true; } +void FormatFactory::markFormatSupportsSubcolumns(const String & name) +{ + auto & target = dict[name].supports_subcolumns; + if (target) + throw Exception("FormatFactory: Format " + name + " is already marked as supporting subcolumns", ErrorCodes::LOGICAL_ERROR); + target = true; +} + +bool FormatFactory::checkIfFormatSupportsSubcolumns(const String & name) const +{ + const auto & target = getCreators(name); + return target.supports_subcolumns; +} bool FormatFactory::checkIfFormatSupportsSubsetOfColumns(const String & name) const { diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 6d76e2f913f..7af43664a50 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -118,6 +118,7 @@ private: SchemaReaderCreator schema_reader_creator; ExternalSchemaReaderCreator external_schema_reader_creator; bool supports_parallel_formatting{false}; + bool supports_subcolumns{false}; bool supports_subset_of_columns{false}; NonTrivialPrefixAndSuffixChecker non_trivial_prefix_and_suffix_checker; AppendSupportChecker append_support_checker; @@ -205,8 +206,10 @@ public: void registerExternalSchemaReader(const String & name, ExternalSchemaReaderCreator external_schema_reader_creator); void markOutputFormatSupportsParallelFormatting(const String & name); + void markFormatSupportsSubcolumns(const String & name); void markFormatSupportsSubsetOfColumns(const String & name); + bool checkIfFormatSupportsSubcolumns(const String & name) const; bool checkIfFormatSupportsSubsetOfColumns(const String & name) const; bool checkIfFormatHasSchemaReader(const String & name) const; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index c7c9bfc816c..ad2f05a5819 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -79,6 +79,8 @@ struct FormatSettings UInt64 input_allow_errors_num = 0; Float32 input_allow_errors_ratio = 0; + UInt64 max_binary_string_size = 0; + struct { UInt64 row_group_size = 1000000; diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index c84e23da85b..93374f933b7 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -29,9 +29,9 @@ list (APPEND PRIVATE_LIBS ch_contrib::zlib boost::filesystem divide_impl + ch_contrib::xxHash ) - if (TARGET ch_rust::blake3) list (APPEND PUBLIC_LIBS ch_rust::blake3 @@ -66,8 +66,6 @@ if (TARGET ch_contrib::base64) list (APPEND PRIVATE_LIBS ch_contrib::base64) endif() -list (APPEND PRIVATE_LIBS ch_contrib::lz4) - if (ENABLE_NLP) list (APPEND PRIVATE_LIBS ch_contrib::cld2) endif() diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index aa1e1f86569..e9a4e357b7e 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -1190,9 +1190,9 @@ struct ToRelativeHourNumImpl static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { if constexpr (precision_ == ResultPrecision::Extended) - return static_cast(time_zone.toStableRelativeHourNum(static_cast(t))); + return static_cast(time_zone.toStableRelativeHourNum(static_cast(t))); else - return static_cast(time_zone.toRelativeHourNum(static_cast(t))); + return static_cast(time_zone.toRelativeHourNum(static_cast(t))); } static inline auto execute(Int32 d, const DateLUTImpl & time_zone) { @@ -1226,7 +1226,7 @@ struct ToRelativeMinuteNumImpl } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { - return static_cast(time_zone.toRelativeMinuteNum(static_cast(t))); + return static_cast(time_zone.toRelativeMinuteNum(static_cast(t))); } static inline auto execute(Int32 d, const DateLUTImpl & time_zone) { diff --git a/src/Functions/FunctionsCodingIP.cpp b/src/Functions/FunctionsCodingIP.cpp index eaf62e232f7..3fea5e9d898 100644 --- a/src/Functions/FunctionsCodingIP.cpp +++ b/src/Functions/FunctionsCodingIP.cpp @@ -232,8 +232,8 @@ public: private: static bool isIPv4Mapped(const UInt8 * address) { - return (unalignedLoad(address) == 0) && - ((unalignedLoad(address + 8) & 0x00000000FFFFFFFFull) == 0x00000000FFFF0000ull); + return (unalignedLoadLE(address) == 0) && + ((unalignedLoadLE(address + 8) & 0x00000000FFFFFFFFull) == 0x00000000FFFF0000ull); } static void cutAddress(const unsigned char * address, char *& dst, UInt8 zeroed_tail_bytes_count) @@ -514,7 +514,11 @@ private: static void mapIPv4ToIPv6(UInt32 in, UInt8 * buf) { unalignedStore(buf, 0); - unalignedStore(buf + 8, 0x00000000FFFF0000ull | (static_cast(ntohl(in)) << 32)); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + unalignedStoreLE(buf + 8, 0x00000000FFFF0000ull | (static_cast(ntohl(in)) << 32)); +#else + unalignedStoreLE(buf + 8, 0x00000000FFFF0000ull | (static_cast(__builtin_bswap32(ntohl(in))) << 32)); +#endif } }; diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 208da8a78fe..c6bb45ddbde 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -2148,7 +2148,13 @@ struct ToNumberMonotonicity return { .is_monotonic = true, .is_always_monotonic = true }; /// If converting from Float, for monotonicity, arguments must fit in range of result type. - if (WhichDataType(type).isFloat()) + bool is_type_float = false; + if (const auto * low_cardinality = typeid_cast(&type)) + is_type_float = WhichDataType(low_cardinality->getDictionaryType()).isFloat(); + else + is_type_float = WhichDataType(type).isFloat(); + + if (is_type_float) { if (left.isNull() || right.isNull()) return {}; @@ -2297,6 +2303,10 @@ struct ToStringMonotonicity if (const auto * low_cardinality_type = checkAndGetDataType(type_ptr)) type_ptr = low_cardinality_type->getDictionaryType().get(); + /// Order on enum values (which is the order on integers) is completely arbitrary in respect to the order on strings. + if (WhichDataType(type).isEnum()) + return not_monotonic; + /// `toString` function is monotonous if the argument is Date or Date32 or DateTime or String, or non-negative numbers with the same number of symbols. if (checkDataTypes(type_ptr)) return positive; diff --git a/src/Functions/FunctionsDecimalArithmetics.cpp b/src/Functions/FunctionsDecimalArithmetics.cpp new file mode 100644 index 00000000000..f275f169914 --- /dev/null +++ b/src/Functions/FunctionsDecimalArithmetics.cpp @@ -0,0 +1,17 @@ +#include +#include + +namespace DB +{ +REGISTER_FUNCTION(DivideDecimals) +{ + factory.registerFunction>(Documentation( + "Decimal division with given precision. Slower than simple `divide`, but has controlled precision and no sound overflows")); +} + +REGISTER_FUNCTION(MultiplyDecimals) +{ + factory.registerFunction>(Documentation( + "Decimal multiplication with given precision. Slower than simple `divide`, but has controlled precision and no sound overflows")); +} +} diff --git a/src/Functions/FunctionsDecimalArithmetics.h b/src/Functions/FunctionsDecimalArithmetics.h new file mode 100644 index 00000000000..9806d13ed30 --- /dev/null +++ b/src/Functions/FunctionsDecimalArithmetics.h @@ -0,0 +1,457 @@ +#pragma once +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int DECIMAL_OVERFLOW; + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_DIVISION; +} + + +struct DecimalOpHelpers +{ + /* These functions perform main arithmetic logic. + * As soon as intermediate results may not fit Decimal256 (e.g. 1e36, scale 10), + * we may not operate with Decimals. Later on this big number may be shrunk (e.g. result scale is 0 in the case above). + * That's why we need to store intermediate results in a flexible extendable storage (here we use std::vector) + * Here we operate on numbers using simple digit arithmetic. + * This is the reason these functions are slower than traditional ones. + * + * Here and below we use UInt8 for storing digits (0-9 range with maximum carry of 9 will definitely fit this) + */ + static std::vector multiply(const std::vector & num1, const std::vector & num2) + { + UInt16 const len1 = num1.size(); + UInt16 const len2 = num2.size(); + if (len1 == 0 || len2 == 0) + return {0}; + + std::vector result(len1 + len2, 0); + UInt16 i_n1 = 0; + UInt16 i_n2; + + for (Int32 i = len1 - 1; i >= 0; --i) + { + UInt16 carry = 0; + i_n2 = 0; + for (Int32 j = len2 - 1; j >= 0; --j) + { + if (unlikely(i_n1 + i_n2 >= len1 + len2)) + throw DB::Exception("Numeric overflow: result bigger that Decimal256", ErrorCodes::DECIMAL_OVERFLOW); + UInt16 sum = num1[i] * num2[j] + result[i_n1 + i_n2] + carry; + carry = sum / 10; + result[i_n1 + i_n2] = sum % 10; + ++i_n2; + } + + if (carry > 0) + { + if (unlikely(i_n1 + i_n2 >= len1 + len2)) + throw DB::Exception("Numeric overflow: result bigger that Decimal256", ErrorCodes::DECIMAL_OVERFLOW); + result[i_n1 + i_n2] += carry; + } + + ++i_n1; + } + + // Maximum Int32 value exceeds 2 billion, we can safely use it for array length storing + Int32 i = static_cast(result.size() - 1); + + while (i >= 0 && result[i] == 0) + { + result.pop_back(); + --i; + } + if (i == -1) + return {0}; + + std::reverse(result.begin(), result.end()); + return result; + } + + static std::vector divide(const std::vector & number, const Int256 & divisor) + { + std::vector result; + const auto max_index = number.size() - 1; + + UInt16 idx = 0; + Int256 temp = 0; + + while (temp < divisor && max_index > idx) + { + temp = temp * 10 + number[idx]; + ++idx; + } + + if (unlikely(temp == 0)) + return {0}; + + while (max_index >= idx) + { + result.push_back(temp / divisor); + temp = (temp % divisor) * 10 + number[idx]; + ++idx; + } + result.push_back(temp / divisor); + + return result; + } + + static std::vector toDigits(Int256 x) + { + std::vector result; + if (x >= 10) + result = toDigits(x / 10); + + result.push_back(x % 10); + return result; + } + + static UInt256 fromDigits(const std::vector & digits) + { + Int256 result = 0; + Int256 scale = 0; + for (auto i = digits.rbegin(); i != digits.rend(); ++i) + { + result += DecimalUtils::scaleMultiplier(scale) * (*i); + ++scale; + } + return result; + } +}; + + +struct DivideDecimalsImpl +{ + static constexpr auto name = "divideDecimal"; + + template + static inline Decimal256 + execute(FirstType a, SecondType b, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale) + { + if (b.value == 0) + throw DB::Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION); + if (a.value == 0) + return Decimal256(0); + + Int256 sign_a = a.value < 0 ? -1 : 1; + Int256 sign_b = b.value < 0 ? -1 : 1; + + std::vector a_digits = DecimalOpHelpers::toDigits(a.value * sign_a); + + while (scale_a < scale_b + result_scale) + { + a_digits.push_back(0); + ++scale_a; + } + + while (scale_a > scale_b + result_scale && !a_digits.empty()) + { + a_digits.pop_back(); + --scale_a; + } + + if (a_digits.empty()) + return Decimal256(0); + + std::vector divided = DecimalOpHelpers::divide(a_digits, b.value * sign_b); + + if (divided.size() > DecimalUtils::max_precision) + throw DB::Exception("Numeric overflow: result bigger that Decimal256", ErrorCodes::DECIMAL_OVERFLOW); + return Decimal256(sign_a * sign_b * DecimalOpHelpers::fromDigits(divided)); + } +}; + + +struct MultiplyDecimalsImpl +{ + static constexpr auto name = "multiplyDecimal"; + + template + static inline Decimal256 + execute(FirstType a, SecondType b, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale) + { + if (a.value == 0 || b.value == 0) + return Decimal256(0); + + Int256 sign_a = a.value < 0 ? -1 : 1; + Int256 sign_b = b.value < 0 ? -1 : 1; + + std::vector a_digits = DecimalOpHelpers::toDigits(a.value * sign_a); + std::vector b_digits = DecimalOpHelpers::toDigits(b.value * sign_b); + + std::vector multiplied = DecimalOpHelpers::multiply(a_digits, b_digits); + + UInt16 product_scale = scale_a + scale_b; + while (product_scale < result_scale) + { + multiplied.push_back(0); + ++product_scale; + } + + while (product_scale > result_scale&& !multiplied.empty()) + { + multiplied.pop_back(); + --product_scale; + } + + if (multiplied.empty()) + return Decimal256(0); + + if (multiplied.size() > DecimalUtils::max_precision) + throw DB::Exception("Numeric overflow: result bigger that Decimal256", ErrorCodes::DECIMAL_OVERFLOW); + + return Decimal256(sign_a * sign_b * DecimalOpHelpers::fromDigits(multiplied)); + } +}; + + +template +struct Processor +{ + const Transform transform; + + explicit Processor(Transform transform_) + : transform(std::move(transform_)) + {} + + template + void NO_INLINE + vectorConstant(const FirstArgVectorType & vec_first, const SecondArgType second_value, + PaddedPODArray & vec_to, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale) const + { + size_t size = vec_first.size(); + vec_to.resize(size); + + for (size_t i = 0; i < size; ++i) + vec_to[i] = transform.execute(vec_first[i], second_value, scale_a, scale_b, result_scale); + } + + template + void NO_INLINE + vectorVector(const FirstArgVectorType & vec_first, const SecondArgVectorType & vec_second, + PaddedPODArray & vec_to, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale) const + { + size_t size = vec_first.size(); + vec_to.resize(size); + + for (size_t i = 0; i < size; ++i) + vec_to[i] = transform.execute(vec_first[i], vec_second[i], scale_a, scale_b, result_scale); + } + + template + void NO_INLINE + constantVector(const FirstArgType & first_value, const SecondArgVectorType & vec_second, + PaddedPODArray & vec_to, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale) const + { + size_t size = vec_second.size(); + vec_to.resize(size); + + for (size_t i = 0; i < size; ++i) + vec_to[i] = transform.execute(first_value, vec_second[i], scale_a, scale_b, result_scale); + } +}; + + +template +struct DecimalArithmeticsImpl +{ + static ColumnPtr execute(Transform transform, const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) + { + using FirstArgValueType = typename FirstArgType::FieldType; + using FirstArgColumnType = typename FirstArgType::ColumnType; + using SecondArgValueType = typename SecondArgType::FieldType; + using SecondArgColumnType = typename SecondArgType::ColumnType; + using ResultColumnType = typename ResultType::ColumnType; + + UInt16 scale_a = getDecimalScale(*arguments[0].type); + UInt16 scale_b = getDecimalScale(*arguments[1].type); + UInt16 result_scale = getDecimalScale(*result_type->getPtr()); + + auto op = Processor{std::move(transform)}; + + auto result_col = result_type->createColumn(); + auto col_to = assert_cast(result_col.get()); + + const auto * first_col = checkAndGetColumn(arguments[0].column.get()); + const auto * second_col = checkAndGetColumn(arguments[1].column.get()); + const auto * first_col_const = typeid_cast(arguments[0].column.get()); + const auto * second_col_const = typeid_cast(arguments[1].column.get()); + + if (first_col) + { + if (second_col_const) + op.vectorConstant(first_col->getData(), second_col_const->template getValue(), col_to->getData(), scale_a, scale_b, result_scale); + else + op.vectorVector(first_col->getData(), second_col->getData(), col_to->getData(), scale_a, scale_b, result_scale); + } + else if (first_col_const) + { + op.constantVector(first_col_const->template getValue(), second_col->getData(), col_to->getData(), scale_a, scale_b, result_scale); + } + else + { + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", + arguments[0].column->getName(), Transform::name); + } + + return result_col; + } +}; + + +template +class FunctionsDecimalArithmetics : public IFunction +{ +public: + static constexpr auto name = Transform::name; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override + { + return name; + } + + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 2 && arguments.size() != 3) + throw Exception("Number of arguments for function " + getName() + " does not match: 2 or 3 expected", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (!isDecimal(arguments[0].type) || !isDecimal(arguments[1].type)) + throw Exception("Arguments for " + getName() + " function must be Decimal", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + UInt8 scale = std::max(getDecimalScale(*arguments[0].type->getPtr()), getDecimalScale(*arguments[1].type->getPtr())); + + if (arguments.size() == 3) + { + WhichDataType which_scale(arguments[2].type.get()); + + if (!which_scale.isUInt8()) + throw Exception( + "Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + + ". Should be constant UInt8 from range[0, 76]", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + const ColumnConst * scale_column = checkAndGetColumnConst(arguments[2].column.get()); + + if (!scale_column) + throw Exception( + "Illegal column of third argument of function " + getName() + ". Should be constant UInt8", + ErrorCodes::ILLEGAL_COLUMN); + + scale = scale_column->getValue(); + } + + /** + At compile time, result is unknown. We only know the Scale (number of fractional digits) at runtime. + Also nothing is known about size of whole part. + As in simple division/multiplication for decimals, we scale the result up, but is is explicit here and no downscale is performed. + It guarantees that result will have given scale and it can also be MANUALLY converted to other decimal types later. + **/ + if (scale > DecimalUtils::max_precision) + throw Exception("Illegal value of third argument of function " + this->getName() + ": must be integer in range [0, 76]", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(DecimalUtils::max_precision, scale); + } + + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {2}; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override + { + return resolveOverload(arguments, result_type); + } + +private: + //long resolver to call proper templated func + ColumnPtr resolveOverload(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const + { + WhichDataType which_dividend(arguments[0].type.get()); + WhichDataType which_divisor(arguments[1].type.get()); + if (which_dividend.isDecimal32()) + { + using DividendType = DataTypeDecimal32; + if (which_divisor.isDecimal32()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal64()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal128()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal256()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + } + + else if (which_dividend.isDecimal64()) + { + using DividendType = DataTypeDecimal64; + if (which_divisor.isDecimal32()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal64()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal128()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal256()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + + } + + else if (which_dividend.isDecimal128()) + { + using DividendType = DataTypeDecimal128; + if (which_divisor.isDecimal32()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal64()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal128()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal256()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + + } + + else if (which_dividend.isDecimal256()) + { + using DividendType = DataTypeDecimal256; + if (which_divisor.isDecimal32()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal64()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal128()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + else if (which_divisor.isDecimal256()) + return DecimalArithmeticsImpl::execute(Transform{}, arguments, result_type); + + } + + // the compiler is happy now + return nullptr; + } +}; + +} + diff --git a/src/Functions/FunctionsHashing.cpp b/src/Functions/FunctionsHashing.cpp index fb631deb4b1..8f616b0be94 100644 --- a/src/Functions/FunctionsHashing.cpp +++ b/src/Functions/FunctionsHashing.cpp @@ -39,6 +39,13 @@ REGISTER_FUNCTION(Hashing) factory.registerFunction(); factory.registerFunction(); + factory.registerFunction( + { + "Calculates value of XXH3 64-bit hash function. Refer to https://github.com/Cyan4973/xxHash for detailed documentation.", + Documentation::Examples{{"hash", "SELECT xxh3('ClickHouse')"}}, + Documentation::Categories{"Hash"} + }, + FunctionFactory::CaseSensitive); factory.registerFunction(); diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index ec0a489471b..ee5f3ea86b5 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -3,12 +3,18 @@ #include #include #include +#include #include #include -#include #include "config.h" +#ifdef __clang__ +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wused-but-marked-unused" +#endif +#include + #if USE_BLAKE3 # include #endif @@ -17,7 +23,6 @@ #include #include #include -#include #if USE_SSL # include @@ -588,7 +593,7 @@ struct ImplXxHash32 static constexpr auto name = "xxHash32"; using ReturnType = UInt32; - static auto apply(const char * s, const size_t len) { return XXH32(s, len, 0); } + static auto apply(const char * s, const size_t len) { return XXH_INLINE_XXH32(s, len, 0); } /** * With current implementation with more than 1 arguments it will give the results * non-reproducible from outside of CH. @@ -609,7 +614,24 @@ struct ImplXxHash64 using ReturnType = UInt64; using uint128_t = CityHash_v1_0_2::uint128; - static auto apply(const char * s, const size_t len) { return XXH64(s, len, 0); } + static auto apply(const char * s, const size_t len) { return XXH_INLINE_XXH64(s, len, 0); } + + /* + With current implementation with more than 1 arguments it will give the results + non-reproducible from outside of CH. (see comment on ImplXxHash32). + */ + static auto combineHashes(UInt64 h1, UInt64 h2) { return CityHash_v1_0_2::Hash128to64(uint128_t(h1, h2)); } + + static constexpr bool use_int_hash_for_pods = false; +}; + +struct ImplXXH3 +{ + static constexpr auto name = "xxh3"; + using ReturnType = UInt64; + using uint128_t = CityHash_v1_0_2::uint128; + + static auto apply(const char * s, const size_t len) { return XXH_INLINE_XXH3_64bits(s, len); } /* With current implementation with more than 1 arguments it will give the results @@ -1508,7 +1530,12 @@ using FunctionHiveHash = FunctionAnyHash; using FunctionXxHash32 = FunctionAnyHash; using FunctionXxHash64 = FunctionAnyHash; +using FunctionXXH3 = FunctionAnyHash; using FunctionWyHash64 = FunctionAnyHash; using FunctionBLAKE3 = FunctionStringHashFixedString; } + +#ifdef __clang__ +# pragma clang diagnostic pop +#endif diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 0f294023cdb..fc1a353a873 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -382,7 +382,7 @@ protected: */ virtual bool useDefaultImplementationForSparseColumns() const { return true; } - // /// If it isn't, will convert all ColumnLowCardinality arguments to full columns. + /// If it isn't, will convert all ColumnLowCardinality arguments to full columns. virtual bool canBeExecutedOnLowCardinalityDictionary() const { return true; } private: diff --git a/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.cpp b/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.cpp index 9bb0abc6369..d78a8623a18 100644 --- a/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.cpp @@ -4,11 +4,14 @@ #include #include +#include +#include #include #include #include #include #include +#include "Parsers/ASTColumnDeclaration.h" namespace DB @@ -19,24 +22,106 @@ namespace ErrorCodes extern const int UNSUPPORTED_METHOD; } -void UserDefinedSQLFunctionMatcher::visit(ASTPtr & ast, Data &) +void UserDefinedSQLFunctionVisitor::visit(ASTPtr & ast) { - auto * function = ast->as(); - if (!function) + const auto visit_child_with_shared_ptr = [&](ASTPtr & child) + { + if (!child) + return; + + auto * old_value = child.get(); + visit(child); + + // child did not change + if (old_value == child.get()) + return; + + // child changed, we need to modify it in the list of children of the parent also + for (auto & current_child : ast->children) + { + if (current_child.get() == old_value) + current_child = child; + } + }; + + if (auto * col_decl = ast->as()) + { + visit_child_with_shared_ptr(col_decl->default_expression); + visit_child_with_shared_ptr(col_decl->ttl); + return; + } + + if (auto * storage = ast->as()) + { + const auto visit_child = [&](IAST * & child) + { + if (!child) + return; + + if (const auto * function = child->template as()) + { + std::unordered_set udf_in_replace_process; + auto replace_result = tryToReplaceFunction(*function, udf_in_replace_process); + if (replace_result) + ast->setOrReplace(child, replace_result); + } + + visit(child); + }; + + visit_child(storage->partition_by); + visit_child(storage->primary_key); + visit_child(storage->order_by); + visit_child(storage->sample_by); + visit_child(storage->ttl_table); + + return; + } + + if (auto * alter = ast->as()) + { + visit_child_with_shared_ptr(alter->col_decl); + visit_child_with_shared_ptr(alter->column); + visit_child_with_shared_ptr(alter->partition); + visit_child_with_shared_ptr(alter->order_by); + visit_child_with_shared_ptr(alter->sample_by); + visit_child_with_shared_ptr(alter->index_decl); + visit_child_with_shared_ptr(alter->index); + visit_child_with_shared_ptr(alter->constraint_decl); + visit_child_with_shared_ptr(alter->constraint); + visit_child_with_shared_ptr(alter->projection_decl); + visit_child_with_shared_ptr(alter->projection); + visit_child_with_shared_ptr(alter->predicate); + visit_child_with_shared_ptr(alter->update_assignments); + visit_child_with_shared_ptr(alter->values); + visit_child_with_shared_ptr(alter->ttl); + visit_child_with_shared_ptr(alter->select); + + return; + } + + if (const auto * function = ast->template as()) + { + std::unordered_set udf_in_replace_process; + auto replace_result = tryToReplaceFunction(*function, udf_in_replace_process); + if (replace_result) + ast = replace_result; + } + + for (auto & child : ast->children) + visit(child); +} + +void UserDefinedSQLFunctionVisitor::visit(IAST * ast) +{ + if (!ast) return; - std::unordered_set udf_in_replace_process; - auto replace_result = tryToReplaceFunction(*function, udf_in_replace_process); - if (replace_result) - ast = replace_result; + for (auto & child : ast->children) + visit(child); } -bool UserDefinedSQLFunctionMatcher::needChildVisit(const ASTPtr &, const ASTPtr &) -{ - return true; -} - -ASTPtr UserDefinedSQLFunctionMatcher::tryToReplaceFunction(const ASTFunction & function, std::unordered_set & udf_in_replace_process) +ASTPtr UserDefinedSQLFunctionVisitor::tryToReplaceFunction(const ASTFunction & function, std::unordered_set & udf_in_replace_process) { if (udf_in_replace_process.find(function.name) != udf_in_replace_process.end()) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, diff --git a/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.h b/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.h index 686594c088f..c8cbf396707 100644 --- a/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.h +++ b/src/Functions/UserDefined/UserDefinedSQLFunctionVisitor.h @@ -19,26 +19,14 @@ class ASTFunction; * After applying visitor: * SELECT number + 1 FROM system.numbers LIMIT 10; */ -class UserDefinedSQLFunctionMatcher +class UserDefinedSQLFunctionVisitor { public: - using Visitor = InDepthNodeVisitor; - - struct Data - { - }; - - static void visit(ASTPtr & ast, Data & data); - static bool needChildVisit(const ASTPtr & node, const ASTPtr & child); - + static void visit(ASTPtr & ast); private: - static void visit(ASTFunction & func, const Data & data); - + static void visit(IAST *); static ASTPtr tryToReplaceFunction(const ASTFunction & function, std::unordered_set & udf_in_replace_process); }; -/// Visits AST nodes and collect their aliases in one map (with links to source nodes). -using UserDefinedSQLFunctionVisitor = UserDefinedSQLFunctionMatcher::Visitor; - } diff --git a/src/Functions/array/arrayFirstLast.cpp b/src/Functions/array/arrayFirstLast.cpp index 8160234a6b0..fa72ecba161 100644 --- a/src/Functions/array/arrayFirstLast.cpp +++ b/src/Functions/array/arrayFirstLast.cpp @@ -43,6 +43,16 @@ struct ArrayFirstLastImpl return array_element; } + static ColumnPtr createNullableColumn(MutableColumnPtr && column, ColumnUInt8::MutablePtr && null_map) + { + if (auto * nullable_column = typeid_cast(column.get())) + { + nullable_column->applyNullMap(*null_map); + return std::move(column); + } + return ColumnNullable::create(std::move(column), std::move(null_map)); + } + static ColumnPtr execute(const ColumnArray & array, ColumnPtr mapped) { const auto * column_filter = typeid_cast(&*mapped); @@ -94,7 +104,7 @@ struct ArrayFirstLastImpl } if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) - return ColumnNullable::create(std::move(out), std::move(col_null_map_to)); + return createNullableColumn(std::move(out), std::move(col_null_map_to)); return out; } @@ -106,7 +116,7 @@ struct ArrayFirstLastImpl if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) { auto col_null_map_to = ColumnUInt8::create(out->size(), true); - return ColumnNullable::create(std::move(out), std::move(col_null_map_to)); + return createNullableColumn(std::move(out), std::move(col_null_map_to)); } return out; @@ -172,7 +182,7 @@ struct ArrayFirstLastImpl } if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) - return ColumnNullable::create(std::move(out), std::move(col_null_map_to)); + return createNullableColumn(std::move(out), std::move(col_null_map_to)); return out; } diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp index b3f1d5ae460..7471fdacbb5 100644 --- a/src/Functions/translate.cpp +++ b/src/Functions/translate.cpp @@ -27,14 +27,14 @@ struct TranslateImpl const std::string & map_to) { if (map_from.size() != map_to.size()) - throw Exception("Second and trird arguments must be the same length", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Second and third arguments must be the same length", ErrorCodes::BAD_ARGUMENTS); std::iota(map.begin(), map.end(), 0); for (size_t i = 0; i < map_from.size(); ++i) { if (!isASCII(map_from[i]) || !isASCII(map_to[i])) - throw Exception("Second and trird arguments must be ASCII strings", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Second and third arguments must be ASCII strings", ErrorCodes::BAD_ARGUMENTS); map[map_from[i]] = map_to[i]; } @@ -125,7 +125,7 @@ struct TranslateUTF8Impl auto map_to_size = UTF8::countCodePoints(reinterpret_cast(map_to.data()), map_to.size()); if (map_from_size != map_to_size) - throw Exception("Second and trird arguments must be the same length", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Second and third arguments must be the same length", ErrorCodes::BAD_ARGUMENTS); std::iota(map_ascii.begin(), map_ascii.end(), 0); diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 95fa5ed543d..69d75f28960 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include @@ -2234,6 +2235,10 @@ BlocksList Aggregator::prepareBlocksAndFillTwoLevelImpl( auto converter = [&](size_t thread_id, ThreadGroupStatusPtr thread_group) { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); if (thread_group) CurrentThread::attachToIfDetached(thread_group); @@ -2951,6 +2956,10 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari auto merge_bucket = [&bucket_to_blocks, &result, this](Int32 bucket, Arena * aggregates_pool, ThreadGroupStatusPtr thread_group) { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); if (thread_group) CurrentThread::attachToIfDetached(thread_group); diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index bf85affcb90..a61f4bdc530 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -48,15 +48,22 @@ namespace ErrorCodes extern const int TIMEOUT_EXCEEDED; extern const int UNKNOWN_EXCEPTION; extern const int UNKNOWN_FORMAT; + extern const int BAD_ARGUMENTS; } AsynchronousInsertQueue::InsertQuery::InsertQuery(const ASTPtr & query_, const Settings & settings_) - : query(query_->clone()), settings(settings_) + : query(query_->clone()) + , query_str(queryToString(query)) + , settings(settings_) + , hash(calculateHash()) { } AsynchronousInsertQueue::InsertQuery::InsertQuery(const InsertQuery & other) - : query(other.query->clone()), settings(other.settings) + : query(other.query->clone()) + , query_str(other.query_str) + , settings(other.settings) + , hash(other.hash) { } @@ -66,29 +73,33 @@ AsynchronousInsertQueue::InsertQuery::operator=(const InsertQuery & other) if (this != &other) { query = other.query->clone(); + query_str = other.query_str; settings = other.settings; + hash = other.hash; } return *this; } -UInt64 AsynchronousInsertQueue::InsertQuery::Hash::operator()(const InsertQuery & insert_query) const +UInt128 AsynchronousInsertQueue::InsertQuery::calculateHash() const { - SipHash hash; - insert_query.query->updateTreeHash(hash); + SipHash siphash; + query->updateTreeHash(siphash); - for (const auto & setting : insert_query.settings.allChanged()) + for (const auto & setting : settings.allChanged()) { - hash.update(setting.getName()); - applyVisitor(FieldVisitorHash(hash), setting.getValue()); + siphash.update(setting.getName()); + applyVisitor(FieldVisitorHash(siphash), setting.getValue()); } - return hash.get64(); + UInt128 res; + siphash.get128(res); + return res; } bool AsynchronousInsertQueue::InsertQuery::operator==(const InsertQuery & other) const { - return queryToString(query) == queryToString(other.query) && settings == other.settings; + return query_str == other.query_str && settings == other.settings; } AsynchronousInsertQueue::InsertData::Entry::Entry(String && bytes_, String && query_id_) @@ -100,43 +111,31 @@ AsynchronousInsertQueue::InsertData::Entry::Entry(String && bytes_, String && qu void AsynchronousInsertQueue::InsertData::Entry::finish(std::exception_ptr exception_) { - std::lock_guard lock(mutex); - finished = true; + if (finished.exchange(true)) + return; + if (exception_) + { + promise.set_exception(exception_); ProfileEvents::increment(ProfileEvents::FailedAsyncInsertQuery, 1); - exception = exception_; - cv.notify_all(); + } + else + { + promise.set_value(); + } } -bool AsynchronousInsertQueue::InsertData::Entry::wait(const Milliseconds & timeout) const -{ - std::unique_lock lock(mutex); - return cv.wait_for(lock, timeout, [&] { return finished; }); -} - -bool AsynchronousInsertQueue::InsertData::Entry::isFinished() const -{ - std::lock_guard lock(mutex); - return finished; -} - -std::exception_ptr AsynchronousInsertQueue::InsertData::Entry::getException() const -{ - std::lock_guard lock(mutex); - return exception; -} - - -AsynchronousInsertQueue::AsynchronousInsertQueue(ContextPtr context_, size_t pool_size, Milliseconds cleanup_timeout_) +AsynchronousInsertQueue::AsynchronousInsertQueue(ContextPtr context_, size_t pool_size_) : WithContext(context_) - , cleanup_timeout(cleanup_timeout_) + , pool_size(pool_size_) + , queue_shards(pool_size) , pool(pool_size) - , dump_by_first_update_thread(&AsynchronousInsertQueue::busyCheck, this) - , cleanup_thread(&AsynchronousInsertQueue::cleanup, this) { - using namespace std::chrono; + if (!pool_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "pool_size cannot be zero"); - assert(pool_size); + for (size_t i = 0; i < pool_size; ++i) + dump_by_first_update_threads.emplace_back([this, i] { processBatchDeadlines(i); }); } AsynchronousInsertQueue::~AsynchronousInsertQueue() @@ -144,34 +143,31 @@ AsynchronousInsertQueue::~AsynchronousInsertQueue() /// TODO: add a setting for graceful shutdown. LOG_TRACE(log, "Shutting down the asynchronous insertion queue"); - shutdown = true; - { - std::lock_guard lock(deadline_mutex); - are_tasks_available.notify_one(); - } - { - std::lock_guard lock(cleanup_mutex); - cleanup_can_run.notify_one(); - } - assert(dump_by_first_update_thread.joinable()); - dump_by_first_update_thread.join(); + for (size_t i = 0; i < pool_size; ++i) + { + auto & shard = queue_shards[i]; - assert(cleanup_thread.joinable()); - cleanup_thread.join(); + shard.are_tasks_available.notify_one(); + assert(dump_by_first_update_threads[i].joinable()); + dump_by_first_update_threads[i].join(); + + { + std::lock_guard lock(shard.mutex); + + for (auto & [_, elem] : shard.queue) + { + for (const auto & entry : elem.data->entries) + { + entry->finish(std::make_exception_ptr(Exception( + ErrorCodes::TIMEOUT_EXCEEDED, "Wait for async insert timeout exceeded)"))); + } + } + } + } pool.wait(); - - std::lock_guard lock(currently_processing_mutex); - for (const auto & [_, entry] : currently_processing_queries) - { - if (!entry->isFinished()) - entry->finish(std::make_exception_ptr(Exception( - ErrorCodes::TIMEOUT_EXCEEDED, - "Wait for async insert timeout exceeded)"))); - } - LOG_TRACE(log, "Asynchronous insertion queue finished"); } @@ -185,7 +181,7 @@ void AsynchronousInsertQueue::scheduleDataProcessingJob(const InsertQuery & key, }); } -void AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_context) +std::future AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_context) { query = query->clone(); const auto & settings = query_context->getSettingsRef(); @@ -214,97 +210,77 @@ void AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_context) quota->used(QuotaType::WRITTEN_BYTES, bytes.size()); auto entry = std::make_shared(std::move(bytes), query_context->getCurrentQueryId()); + InsertQuery key{query, settings}; + InsertDataPtr data_to_process; + std::future insert_future; + + auto shard_num = key.hash % pool_size; + auto & shard = queue_shards[shard_num]; { - /// Firstly try to get entry from queue without exclusive lock. - std::shared_lock read_lock(rwlock); - if (auto it = queue.find(key); it != queue.end()) + std::lock_guard lock(shard.mutex); + + auto [it, inserted] = shard.iterators.try_emplace(key.hash); + if (inserted) { - pushImpl(std::move(entry), it); - return; + auto now = std::chrono::steady_clock::now(); + auto timeout = now + Milliseconds{key.settings.async_insert_busy_timeout_ms}; + it->second = shard.queue.emplace(timeout, Container{key, std::make_unique()}).first; } + + auto queue_it = it->second; + auto & data = queue_it->second.data; + size_t entry_data_size = entry->bytes.size(); + + assert(data); + data->size_in_bytes += entry_data_size; + data->entries.emplace_back(entry); + insert_future = entry->getFuture(); + + LOG_TRACE(log, "Have {} pending inserts with total {} bytes of data for query '{}'", + data->entries.size(), data->size_in_bytes, key.query_str); + + /// Here we check whether we hit the limit on maximum data size in the buffer. + /// And use setting from query context. + /// It works, because queries with the same set of settings are already grouped together. + if (data->size_in_bytes > key.settings.async_insert_max_data_size) + { + data_to_process = std::move(data); + shard.iterators.erase(it); + shard.queue.erase(queue_it); + } + + CurrentMetrics::add(CurrentMetrics::PendingAsyncInsert); + ProfileEvents::increment(ProfileEvents::AsyncInsertQuery); + ProfileEvents::increment(ProfileEvents::AsyncInsertBytes, entry_data_size); } - std::lock_guard write_lock(rwlock); - auto it = queue.emplace(key, std::make_shared()).first; - pushImpl(std::move(entry), it); + if (data_to_process) + scheduleDataProcessingJob(key, std::move(data_to_process), getContext()); + else + shard.are_tasks_available.notify_one(); + + return insert_future; } -void AsynchronousInsertQueue::pushImpl(InsertData::EntryPtr entry, QueueIterator it) +void AsynchronousInsertQueue::processBatchDeadlines(size_t shard_num) { - auto & [data_mutex, data] = *it->second; - std::lock_guard data_lock(data_mutex); + auto & shard = queue_shards[shard_num]; - if (!data) - { - auto now = std::chrono::steady_clock::now(); - data = std::make_unique(now); - - std::lock_guard lock(deadline_mutex); - deadline_queue.insert({now + Milliseconds{it->first.settings.async_insert_busy_timeout_ms}, it}); - are_tasks_available.notify_one(); - } - - size_t entry_data_size = entry->bytes.size(); - - data->size += entry_data_size; - data->entries.emplace_back(entry); - - { - std::lock_guard currently_processing_lock(currently_processing_mutex); - currently_processing_queries.emplace(entry->query_id, entry); - } - - LOG_TRACE(log, "Have {} pending inserts with total {} bytes of data for query '{}'", - data->entries.size(), data->size, queryToString(it->first.query)); - - /// Here we check whether we hit the limit on maximum data size in the buffer. - /// And use setting from query context! - /// It works, because queries with the same set of settings are already grouped together. - if (data->size > it->first.settings.async_insert_max_data_size) - scheduleDataProcessingJob(it->first, std::move(data), getContext()); - - CurrentMetrics::add(CurrentMetrics::PendingAsyncInsert); - ProfileEvents::increment(ProfileEvents::AsyncInsertQuery); - ProfileEvents::increment(ProfileEvents::AsyncInsertBytes, entry_data_size); -} - -void AsynchronousInsertQueue::waitForProcessingQuery(const String & query_id, const Milliseconds & timeout) -{ - InsertData::EntryPtr entry; - - { - std::lock_guard lock(currently_processing_mutex); - auto it = currently_processing_queries.find(query_id); - if (it == currently_processing_queries.end()) - return; - - entry = it->second; - } - - bool finished = entry->wait(timeout); - - if (!finished) - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Wait for async insert timeout ({} ms) exceeded)", timeout.count()); - - if (auto exception = entry->getException()) - std::rethrow_exception(exception); -} - -void AsynchronousInsertQueue::busyCheck() -{ while (!shutdown) { - std::vector entries_to_flush; + std::vector entries_to_flush; { - std::unique_lock deadline_lock(deadline_mutex); - are_tasks_available.wait_for(deadline_lock, Milliseconds(getContext()->getSettingsRef().async_insert_busy_timeout_ms), [this]() + std::unique_lock lock(shard.mutex); + + shard.are_tasks_available.wait_for(lock, + Milliseconds(getContext()->getSettingsRef().async_insert_busy_timeout_ms), [&shard, this] { if (shutdown) return true; - if (!deadline_queue.empty() && deadline_queue.begin()->first < std::chrono::steady_clock::now()) + if (!shard.queue.empty() && shard.queue.begin()->first < std::chrono::steady_clock::now()) return true; return false; @@ -317,91 +293,22 @@ void AsynchronousInsertQueue::busyCheck() while (true) { - if (deadline_queue.empty() || deadline_queue.begin()->first > now) + if (shard.queue.empty() || shard.queue.begin()->first > now) break; - entries_to_flush.emplace_back(deadline_queue.begin()->second); - deadline_queue.erase(deadline_queue.begin()); + auto it = shard.queue.begin(); + shard.iterators.erase(it->second.key.hash); + + entries_to_flush.emplace_back(std::move(it->second)); + shard.queue.erase(it); } } - std::shared_lock read_lock(rwlock); for (auto & entry : entries_to_flush) - { - auto & [key, elem] = *entry; - std::lock_guard data_lock(elem->mutex); - if (!elem->data) - continue; - - scheduleDataProcessingJob(key, std::move(elem->data), getContext()); - } + scheduleDataProcessingJob(entry.key, std::move(entry.data), getContext()); } } -void AsynchronousInsertQueue::cleanup() -{ - while (true) - { - { - std::unique_lock cleanup_lock(cleanup_mutex); - cleanup_can_run.wait_for(cleanup_lock, Milliseconds(cleanup_timeout), [this]() -> bool { return shutdown; }); - - if (shutdown) - return; - } - - std::vector keys_to_remove; - - { - std::shared_lock read_lock(rwlock); - - for (auto & [key, elem] : queue) - { - std::lock_guard data_lock(elem->mutex); - if (!elem->data) - keys_to_remove.push_back(key); - } - } - - if (!keys_to_remove.empty()) - { - std::lock_guard write_lock(rwlock); - size_t total_removed = 0; - - for (const auto & key : keys_to_remove) - { - auto it = queue.find(key); - if (it != queue.end() && !it->second->data) - { - queue.erase(it); - ++total_removed; - } - } - - if (total_removed) - LOG_TRACE(log, "Removed stale entries for {} queries from asynchronous insertion queue", total_removed); - } - - { - std::vector ids_to_remove; - std::lock_guard lock(currently_processing_mutex); - - for (const auto & [query_id, entry] : currently_processing_queries) - if (entry->isFinished()) - ids_to_remove.push_back(query_id); - - if (!ids_to_remove.empty()) - { - for (const auto & id : ids_to_remove) - currently_processing_queries.erase(id); - - LOG_TRACE(log, "Removed {} finished entries from asynchronous insertion queue", ids_to_remove.size()); - } - } - } -} - - static void appendElementsToLogSafe( AsynchronousInsertLog & log, std::vector elements, @@ -464,7 +371,7 @@ try { current_exception = e.displayText(); LOG_ERROR(log, "Failed parsing for query '{}' with query id {}. {}", - queryToString(key.query), current_entry->query_id, current_exception); + key.query_str, current_entry->query_id, current_exception); for (const auto & column : result_columns) if (column->size() > total_rows) @@ -546,7 +453,7 @@ try completed_executor.execute(); LOG_INFO(log, "Flushed {} rows, {} bytes for query '{}'", - total_rows, total_bytes, queryToString(key.query)); + total_rows, total_bytes, key.query_str); } catch (...) { diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index fcf4e3d98d2..71a3bce235e 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -4,10 +4,7 @@ #include #include #include - -#include -#include - +#include namespace DB { @@ -19,25 +16,29 @@ class AsynchronousInsertQueue : public WithContext public: using Milliseconds = std::chrono::milliseconds; - AsynchronousInsertQueue(ContextPtr context_, size_t pool_size, Milliseconds cleanup_timeout); + AsynchronousInsertQueue(ContextPtr context_, size_t pool_size_); ~AsynchronousInsertQueue(); - void push(ASTPtr query, ContextPtr query_context); - void waitForProcessingQuery(const String & query_id, const Milliseconds & timeout); + std::future push(ASTPtr query, ContextPtr query_context); + size_t getPoolSize() const { return pool_size; } private: struct InsertQuery { + public: ASTPtr query; + String query_str; Settings settings; + UInt128 hash; InsertQuery(const ASTPtr & query_, const Settings & settings_); InsertQuery(const InsertQuery & other); InsertQuery & operator=(const InsertQuery & other); - bool operator==(const InsertQuery & other) const; - struct Hash { UInt64 operator()(const InsertQuery & insert_query) const; }; + + private: + UInt128 calculateHash() const; }; struct InsertData @@ -47,109 +48,84 @@ private: public: const String bytes; const String query_id; - std::chrono::time_point create_time; + const std::chrono::time_point create_time; Entry(String && bytes_, String && query_id_); void finish(std::exception_ptr exception_ = nullptr); - bool wait(const Milliseconds & timeout) const; - bool isFinished() const; - std::exception_ptr getException() const; + std::future getFuture() { return promise.get_future(); } + bool isFinished() const { return finished; } private: - mutable std::mutex mutex; - mutable std::condition_variable cv; - - bool finished = false; - std::exception_ptr exception; + std::promise promise; + std::atomic_bool finished = false; }; - explicit InsertData(std::chrono::steady_clock::time_point now) - : first_update(now) - {} - using EntryPtr = std::shared_ptr; std::list entries; - size_t size = 0; - - /// Timestamp of the first insert into queue, or after the last queue dump. - /// Used to detect for how long the queue is active, so we can dump it by timer. - std::chrono::time_point first_update; + size_t size_in_bytes = 0; }; using InsertDataPtr = std::unique_ptr; - /// A separate container, that holds a data and a mutex for it. - /// When it's needed to process current chunk of data, it can be moved for processing - /// and new data can be recreated without holding a lock during processing. struct Container { - std::mutex mutex; + InsertQuery key; InsertDataPtr data; }; - using Queue = std::unordered_map, InsertQuery::Hash>; - using QueueIterator = Queue::iterator; /// Ordered container - using DeadlineQueue = std::map; + /// Key is a timestamp of the first insert into batch. + /// Used to detect for how long the batch is active, so we can dump it by timer. + using Queue = std::map; + using QueueIterator = Queue::iterator; + using QueueIteratorByKey = std::unordered_map; + struct QueueShard + { + mutable std::mutex mutex; + mutable std::condition_variable are_tasks_available; - mutable std::shared_mutex rwlock; - Queue queue; + Queue queue; + QueueIteratorByKey iterators; + }; - /// This is needed only for using inside cleanup() function and correct signaling about shutdown - mutable std::mutex cleanup_mutex; - mutable std::condition_variable cleanup_can_run; - - mutable std::mutex deadline_mutex; - mutable std::condition_variable are_tasks_available; - DeadlineQueue deadline_queue; - - using QueryIdToEntry = std::unordered_map; - mutable std::mutex currently_processing_mutex; - QueryIdToEntry currently_processing_queries; + const size_t pool_size; + std::vector queue_shards; /// Logic and events behind queue are as follows: - /// - busy_timeout: if queue is active for too long and there are a lot of rapid inserts, then we dump the data, so it doesn't - /// grow for a long period of time and users will be able to select new data in deterministic manner. - /// - stale_timeout: if queue is stale for too long, then we dump the data too, so that users will be able to select the last - /// piece of inserted data. + /// - async_insert_busy_timeout_ms: + /// if queue is active for too long and there are a lot of rapid inserts, then we dump the data, so it doesn't + /// grow for a long period of time and users will be able to select new data in deterministic manner. /// - /// During processing incoming INSERT queries we can also check whether the maximum size of data in buffer is reached (async_insert_max_data_size setting) - /// If so, then again we dump the data. - - const Milliseconds cleanup_timeout; + /// During processing incoming INSERT queries we can also check whether the maximum size of data in buffer is reached + /// (async_insert_max_data_size setting). If so, then again we dump the data. std::atomic shutdown{false}; - ThreadPool pool; /// dump the data only inside this pool. - ThreadFromGlobalPool dump_by_first_update_thread; /// uses busy_timeout and busyCheck() - ThreadFromGlobalPool cleanup_thread; /// uses busy_timeout and cleanup() + /// Dump the data only inside this pool. + ThreadPool pool; + + /// Uses async_insert_busy_timeout_ms and processBatchDeadlines() + std::vector dump_by_first_update_threads; Poco::Logger * log = &Poco::Logger::get("AsynchronousInsertQueue"); - void busyCheck(); - void cleanup(); - - /// Should be called with shared or exclusively locked 'rwlock'. - void pushImpl(InsertData::EntryPtr entry, QueueIterator it); - + void processBatchDeadlines(size_t shard_num); void scheduleDataProcessingJob(const InsertQuery & key, InsertDataPtr data, ContextPtr global_context); + static void processData(InsertQuery key, InsertDataPtr data, ContextPtr global_context); template static void finishWithException(const ASTPtr & query, const std::list & entries, const E & exception); - /// @param timeout - time to wait - /// @return true if shutdown requested - bool waitForShutdown(const Milliseconds & timeout); - public: - auto getQueueLocked() const + auto getQueueLocked(size_t shard_num) const { - std::shared_lock lock(rwlock); - return std::make_pair(std::ref(queue), std::move(lock)); + auto & shard = queue_shards[shard_num]; + std::unique_lock lock(shard.mutex); + return std::make_pair(std::ref(shard.queue), std::move(lock)); } }; diff --git a/src/Interpreters/AsynchronousMetricLog.cpp b/src/Interpreters/AsynchronousMetricLog.cpp index 6176bb781ab..eec5da802a7 100644 --- a/src/Interpreters/AsynchronousMetricLog.cpp +++ b/src/Interpreters/AsynchronousMetricLog.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Interpreters/AsynchronousMetricLog.h b/src/Interpreters/AsynchronousMetricLog.h index 8a19fae29e9..1937aa09dbd 100644 --- a/src/Interpreters/AsynchronousMetricLog.h +++ b/src/Interpreters/AsynchronousMetricLog.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 4653491aac9..2e2f886a50a 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -174,18 +174,15 @@ void SelectStreamFactory::createForShard( } -SelectStreamFactory::ShardPlans SelectStreamFactory::createForShardWithParallelReplicas( +void SelectStreamFactory::createForShardWithParallelReplicas( const Cluster::ShardInfo & shard_info, const ASTPtr & query_ast, const StorageID & main_table, - const ASTPtr & table_function_ptr, - const ThrottlerPtr & throttler, ContextPtr context, UInt32 shard_count, - const std::shared_ptr & storage_limits) + std::vector & local_plans, + Shards & remote_shards) { - SelectStreamFactory::ShardPlans result; - if (auto it = objects_by_shard.find(shard_info.shard_num); it != objects_by_shard.end()) replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, query_ast); @@ -213,8 +210,6 @@ SelectStreamFactory::ShardPlans SelectStreamFactory::createForShardWithParallelR size_t all_replicas_count = shard_info.getRemoteNodeCount(); auto coordinator = std::make_shared(); - auto remote_plan = std::make_unique(); - if (settings.prefer_localhost_replica && shard_info.isLocal()) { @@ -223,48 +218,22 @@ SelectStreamFactory::ShardPlans SelectStreamFactory::createForShardWithParallelR { ++all_replicas_count; - result.local_plan = createLocalPlan( - query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, next_replica_number, all_replicas_count, coordinator); + local_plans.emplace_back(createLocalPlan( + query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, next_replica_number, all_replicas_count, coordinator)); ++next_replica_number; } } - Scalars scalars = context->hasQueryContext() ? context->getQueryContext()->getScalars() : Scalars{}; - scalars.emplace( - "_shard_count", Block{{DataTypeUInt32().createColumnConst(1, shard_count), std::make_shared(), "_shard_count"}}); - auto external_tables = context->getExternalTables(); - - auto shard = Shard{ - .query = query_ast, - .header = header, - .shard_info = shard_info, - .lazy = false, - .local_delay = 0, - }; - if (shard_info.hasRemoteConnections()) - { - auto read_from_remote = std::make_unique( - coordinator, - shard, - header, - processed_stage, - main_table, - table_function_ptr, - context, - throttler, - std::move(scalars), - std::move(external_tables), - &Poco::Logger::get("ReadFromParallelRemoteReplicasStep"), - storage_limits); - - remote_plan->addStep(std::move(read_from_remote)); - remote_plan->addInterpreterContext(context); - result.remote_plan = std::move(remote_plan); - } - - return result; + remote_shards.emplace_back(Shard{ + .query = query_ast, + .header = header, + .shard_info = shard_info, + .lazy = false, + .local_delay = 0, + .coordinator = coordinator, + }); } } diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 8ebddea4988..a8f7d131b15 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -1,12 +1,13 @@ #pragma once -#include -#include -#include -#include #include +#include #include +#include #include +#include +#include +#include namespace DB { @@ -47,6 +48,9 @@ public: /// (When there is a local replica with big delay). bool lazy = false; time_t local_delay = 0; + + /// Set only if parallel reading from replicas is used. + std::shared_ptr coordinator; }; using Shards = std::vector; @@ -76,16 +80,14 @@ public: std::unique_ptr remote_plan; }; - ShardPlans createForShardWithParallelReplicas( + void createForShardWithParallelReplicas( const Cluster::ShardInfo & shard_info, const ASTPtr & query_ast, const StorageID & main_table, - const ASTPtr & table_function_ptr, - const ThrottlerPtr & throttler, ContextPtr context, UInt32 shard_count, - const std::shared_ptr & storage_limits - ); + std::vector & local_plans, + Shards & remote_shards); private: const Block header; diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index e9ec38f3806..6f5de6d6e5a 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -1,19 +1,45 @@ -#include -#include +#include #include -#include +#include #include +#include +#include +#include #include -#include +#include #include -#include #include +#include #include #include #include +#include #include -#include +using namespace DB; + +namespace +{ + +/// We determine output stream sort properties by a local plan (local because otherwise table could be unknown). +/// If no local shard exist for this cluster, no sort properties will be provided, c'est la vie. +auto getRemoteShardsOutputStreamSortingProperties(const std::vector & plans, ContextMutablePtr context) +{ + SortDescription sort_description; + DataStream::SortScope sort_scope = DataStream::SortScope::None; + if (!plans.empty()) + { + if (const auto * step = dynamic_cast(plans.front()->getRootNode()->step.get()); + step && step->getDataStreamTraits().can_enforce_sorting_properties_in_distributed_query) + { + step->adjustSettingsToEnforceSortingPropertiesInDistributedQuery(context); + sort_description = step->getOutputStream().sort_description; + sort_scope = step->getOutputStream().sort_scope; + } + } + return std::make_pair(sort_description, sort_scope); +} +} namespace DB { @@ -190,6 +216,8 @@ void executeQuery( "_shard_count", Block{{DataTypeUInt32().createColumnConst(1, shards), std::make_shared(), "_shard_count"}}); auto external_tables = context->getExternalTables(); + auto && [sort_description, sort_scope] = getRemoteShardsOutputStreamSortingProperties(plans, new_context); + auto plan = std::make_unique(); auto read_from_remote = std::make_unique( std::move(remote_shards), @@ -203,7 +231,9 @@ void executeQuery( std::move(external_tables), log, shards, - query_info.storage_limits); + query_info.storage_limits, + std::move(sort_description), + std::move(sort_scope)); read_from_remote->setStepDescription("Read from remote replica"); plan->addStep(std::move(read_from_remote)); @@ -235,10 +265,13 @@ void executeQueryWithParallelReplicas( const StorageID & main_table, const ASTPtr & table_func_ptr, SelectStreamFactory & stream_factory, - const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info, + const ASTPtr & query_ast, + ContextPtr context, + const SelectQueryInfo & query_info, const ExpressionActionsPtr & sharding_key_expr, const std::string & sharding_key_column_name, - const ClusterPtr & not_optimized_cluster) + const ClusterPtr & not_optimized_cluster, + QueryProcessingStage::Enum processed_stage) { const Settings & settings = context->getSettingsRef(); @@ -261,6 +294,7 @@ void executeQueryWithParallelReplicas( std::vector plans; + SelectStreamFactory::Shards remote_shards; size_t shards = query_info.getCluster()->getShardCount(); for (const auto & shard_info : query_info.getCluster()->getShardsInfo()) @@ -283,18 +317,43 @@ void executeQueryWithParallelReplicas( else query_ast_for_shard = query_ast; - auto shard_plans = stream_factory.createForShardWithParallelReplicas(shard_info, - query_ast_for_shard, main_table, table_func_ptr, throttler, context, - static_cast(shards), query_info.storage_limits); + stream_factory.createForShardWithParallelReplicas( + shard_info, query_ast_for_shard, main_table, context, static_cast(shards), plans, remote_shards); + } - if (!shard_plans.local_plan && !shard_plans.remote_plan) - throw Exception(ErrorCodes::LOGICAL_ERROR, "No plans were generated for reading from shard. This is a bug"); + Scalars scalars = context->hasQueryContext() ? context->getQueryContext()->getScalars() : Scalars{}; + scalars.emplace( + "_shard_count", Block{{DataTypeUInt32().createColumnConst(1, shards), std::make_shared(), "_shard_count"}}); + auto external_tables = context->getExternalTables(); - if (shard_plans.local_plan) - plans.emplace_back(std::move(shard_plans.local_plan)); + if (!remote_shards.empty()) + { + auto new_context = Context::createCopy(context); + auto && [sort_description, sort_scope] = getRemoteShardsOutputStreamSortingProperties(plans, new_context); - if (shard_plans.remote_plan) - plans.emplace_back(std::move(shard_plans.remote_plan)); + for (const auto & shard : remote_shards) + { + auto read_from_remote = std::make_unique( + shard.coordinator, + shard, + shard.header, + processed_stage, + main_table, + table_func_ptr, + new_context, + throttler, + scalars, + external_tables, + &Poco::Logger::get("ReadFromParallelRemoteReplicasStep"), + query_info.storage_limits, + sort_description, + sort_scope); + + auto remote_plan = std::make_unique(); + remote_plan->addStep(std::move(read_from_remote)); + remote_plan->addInterpreterContext(new_context); + plans.emplace_back(std::move(remote_plan)); + } } if (plans.empty()) diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index ac88752ce74..662fe47ca65 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -58,11 +58,13 @@ void executeQueryWithParallelReplicas( const StorageID & main_table, const ASTPtr & table_func_ptr, SelectStreamFactory & stream_factory, - const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info, + const ASTPtr & query_ast, + ContextPtr context, + const SelectQueryInfo & query_info, const ExpressionActionsPtr & sharding_key_expr, const std::string & sharding_key_column_name, - const ClusterPtr & not_optimized_cluster); - + const ClusterPtr & not_optimized_cluster, + QueryProcessingStage::Enum processed_stage); } } diff --git a/src/Interpreters/ConcurrentHashJoin.cpp b/src/Interpreters/ConcurrentHashJoin.cpp index cc79a71245b..6c77539532f 100644 --- a/src/Interpreters/ConcurrentHashJoin.cpp +++ b/src/Interpreters/ConcurrentHashJoin.cpp @@ -161,15 +161,12 @@ bool ConcurrentHashJoin::alwaysReturnsEmptySet() const return true; } -std::shared_ptr ConcurrentHashJoin::getNonJoinedBlocks( +IBlocksStreamPtr ConcurrentHashJoin::getNonJoinedBlocks( const Block & /*left_sample_block*/, const Block & /*result_sample_block*/, UInt64 /*max_block_size*/) const { - if (table_join->strictness() == JoinStrictness::Asof || - table_join->strictness() == JoinStrictness::Semi || - !isRightOrFull(table_join->kind())) - { + if (!JoinCommon::hasNonJoinedBlocks(*table_join)) return {}; - } + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid join type. join kind: {}, strictness: {}", table_join->kind(), table_join->strictness()); } @@ -204,6 +201,7 @@ IColumn::Selector ConcurrentHashJoin::selectDispatchBlock(const Strings & key_co Blocks ConcurrentHashJoin::dispatchBlock(const Strings & key_columns_names, const Block & from_block) { + /// TODO: use JoinCommon::scatterBlockByHash size_t num_shards = hash_joins.size(); size_t num_cols = from_block.columns(); diff --git a/src/Interpreters/ConcurrentHashJoin.h b/src/Interpreters/ConcurrentHashJoin.h index 705e6ba81b7..a00c3ed1326 100644 --- a/src/Interpreters/ConcurrentHashJoin.h +++ b/src/Interpreters/ConcurrentHashJoin.h @@ -47,7 +47,7 @@ public: size_t getTotalByteCount() const override; bool alwaysReturnsEmptySet() const override; bool supportParallelJoin() const override { return true; } - std::shared_ptr + IBlocksStreamPtr getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const override; private: diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 89fc17ab2e3..b9c68488fa6 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -14,11 +14,14 @@ #include #include #include +#include #include #include #include #include +#include + #include "config.h" @@ -1082,4 +1085,53 @@ private: DisksMap getDisksMap(std::lock_guard & lock) const; }; +struct HTTPContext : public IHTTPContext +{ + explicit HTTPContext(ContextPtr context_) + : context(Context::createCopy(context_)) + {} + + uint64_t getMaxHstsAge() const override + { + return context->getSettingsRef().hsts_max_age; + } + + uint64_t getMaxUriSize() const override + { + return context->getSettingsRef().http_max_uri_size; + } + + uint64_t getMaxFields() const override + { + return context->getSettingsRef().http_max_fields; + } + + uint64_t getMaxFieldNameSize() const override + { + return context->getSettingsRef().http_max_field_name_size; + } + + uint64_t getMaxFieldValueSize() const override + { + return context->getSettingsRef().http_max_field_value_size; + } + + uint64_t getMaxChunkSize() const override + { + return context->getSettingsRef().http_max_chunk_size; + } + + Poco::Timespan getReceiveTimeout() const override + { + return context->getSettingsRef().http_receive_timeout; + } + + Poco::Timespan getSendTimeout() const override + { + return context->getSettingsRef().http_send_timeout; + } + + ContextPtr context; +}; + } diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 7ceb0bf3a00..a76b13e5dcf 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -223,6 +223,7 @@ void DatabaseCatalog::shutdownImpl() return it != elem.map.end(); }) == uuid_map.end()); databases.clear(); + referential_dependencies.clear(); view_dependencies.clear(); } @@ -473,13 +474,8 @@ void DatabaseCatalog::updateDatabaseName(const String & old_name, const String & for (const auto & table_name : tables_in_database) { - QualifiedTableName new_table_name{new_name, table_name}; - auto dependencies = tryRemoveLoadingDependenciesUnlocked(QualifiedTableName{old_name, table_name}, /* check_dependencies */ false); - DependenciesInfos new_info; - for (const auto & dependency : dependencies) - new_info[dependency].dependent_database_objects.insert(new_table_name); - new_info[new_table_name].dependencies = std::move(dependencies); - mergeDependenciesGraphs(loading_dependencies, new_info); + auto dependencies = referential_dependencies.removeDependencies(StorageID{old_name, table_name}, /* remove_isolated_tables= */ true); + referential_dependencies.addDependencies(StorageID{new_name, table_name}, dependencies); } } @@ -648,7 +644,10 @@ bool DatabaseCatalog::hasUUIDMapping(const UUID & uuid) std::unique_ptr DatabaseCatalog::database_catalog; DatabaseCatalog::DatabaseCatalog(ContextMutablePtr global_context_) - : WithMutableContext(global_context_), log(&Poco::Logger::get("DatabaseCatalog")) + : WithMutableContext(global_context_) + , referential_dependencies{"ReferentialDeps"} + , view_dependencies{"ViewDeps"} + , log(&Poco::Logger::get("DatabaseCatalog")) { } @@ -692,39 +691,33 @@ DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, ContextPt return getDatabase(resolved_database); } -void DatabaseCatalog::addDependency(const StorageID & from, const StorageID & where) +void DatabaseCatalog::addViewDependency(const StorageID & source_table_id, const StorageID & view_id) { std::lock_guard lock{databases_mutex}; - // FIXME when loading metadata storage may not know UUIDs of it's dependencies, because they are not loaded yet, - // so UUID of `from` is not used here. (same for remove, get and update) - view_dependencies[{from.getDatabaseName(), from.getTableName()}].insert(where); + view_dependencies.addDependency(source_table_id, view_id); } -void DatabaseCatalog::removeDependency(const StorageID & from, const StorageID & where) +void DatabaseCatalog::removeViewDependency(const StorageID & source_table_id, const StorageID & view_id) { std::lock_guard lock{databases_mutex}; - view_dependencies[{from.getDatabaseName(), from.getTableName()}].erase(where); + view_dependencies.removeDependency(source_table_id, view_id, /* remove_isolated_tables= */ true); } -Dependencies DatabaseCatalog::getDependencies(const StorageID & from) const +std::vector DatabaseCatalog::getDependentViews(const StorageID & source_table_id) const { std::lock_guard lock{databases_mutex}; - auto iter = view_dependencies.find({from.getDatabaseName(), from.getTableName()}); - if (iter == view_dependencies.end()) - return {}; - return Dependencies(iter->second.begin(), iter->second.end()); + return view_dependencies.getDependencies(source_table_id); } -void -DatabaseCatalog::updateDependency(const StorageID & old_from, const StorageID & old_where, const StorageID & new_from, - const StorageID & new_where) +void DatabaseCatalog::updateViewDependency(const StorageID & old_source_table_id, const StorageID & old_view_id, + const StorageID & new_source_table_id, const StorageID & new_view_id) { std::lock_guard lock{databases_mutex}; - if (!old_from.empty()) - view_dependencies[{old_from.getDatabaseName(), old_from.getTableName()}].erase(old_where); - if (!new_from.empty()) - view_dependencies[{new_from.getDatabaseName(), new_from.getTableName()}].insert(new_where); + if (!old_source_table_id.empty()) + view_dependencies.removeDependency(old_source_table_id, old_view_id, /* remove_isolated_tables= */ true); + if (!new_source_table_id.empty()) + view_dependencies.addDependency(new_source_table_id, new_view_id); } DDLGuardPtr DatabaseCatalog::getDDLGuard(const String & database, const String & table) @@ -869,6 +862,8 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr { chassert(hasUUIDMapping(table_id.uuid)); drop_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + /// Do not postpone removal of in-memory tables + ignore_delay = ignore_delay || !table->storesDataOnDisk(); table->is_dropped = true; } else @@ -1015,7 +1010,7 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) for (const auto & [disk_name, disk] : getContext()->getDisksMap()) { String data_path = "store/" + getPathForUUID(table.table_id.uuid); - if (!disk->exists(data_path) || disk->isReadOnly()) + if (disk->isReadOnly() || !disk->exists(data_path)) continue; LOG_INFO(log, "Removing data directory {} of dropped table {} from disk {}", data_path, table.table_id.getNameForLogs(), disk_name); @@ -1048,121 +1043,79 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid) }); } -void DatabaseCatalog::addLoadingDependencies(const QualifiedTableName & table, TableNamesSet && dependencies) -{ - DependenciesInfos new_info; - for (const auto & dependency : dependencies) - new_info[dependency].dependent_database_objects.insert(table); - new_info[table].dependencies = std::move(dependencies); - addLoadingDependencies(new_info); -} - -void DatabaseCatalog::addLoadingDependencies(const DependenciesInfos & new_infos) +void DatabaseCatalog::addDependencies(const StorageID & table_id, const std::vector & dependencies) { std::lock_guard lock{databases_mutex}; - mergeDependenciesGraphs(loading_dependencies, new_infos); + referential_dependencies.addDependencies(table_id, dependencies); } -DependenciesInfo DatabaseCatalog::getLoadingDependenciesInfo(const StorageID & table_id) const +void DatabaseCatalog::addDependencies(const QualifiedTableName & table_name, const TableNamesSet & dependencies) { std::lock_guard lock{databases_mutex}; - auto it = loading_dependencies.find(table_id.getQualifiedName()); - if (it == loading_dependencies.end()) - return {}; - return it->second; + referential_dependencies.addDependencies(table_name, dependencies); } -TableNamesSet DatabaseCatalog::tryRemoveLoadingDependencies(const StorageID & table_id, bool check_dependencies, bool is_drop_database) +void DatabaseCatalog::addDependencies(const TablesDependencyGraph & extra_graph) { - QualifiedTableName removing_table = table_id.getQualifiedName(); std::lock_guard lock{databases_mutex}; - return tryRemoveLoadingDependenciesUnlocked(removing_table, check_dependencies, is_drop_database); + referential_dependencies.mergeWith(extra_graph); } -TableNamesSet DatabaseCatalog::tryRemoveLoadingDependenciesUnlocked(const QualifiedTableName & removing_table, bool check_dependencies, bool is_drop_database) +std::vector DatabaseCatalog::getDependencies(const StorageID & table_id) const { - auto it = loading_dependencies.find(removing_table); - if (it == loading_dependencies.end()) - return {}; + std::lock_guard lock{databases_mutex}; + return referential_dependencies.getDependencies(table_id); +} - TableNamesSet & dependent = it->second.dependent_database_objects; - if (!dependent.empty()) - { - if (check_dependencies) - checkTableCanBeRemovedOrRenamedImpl(dependent, removing_table, is_drop_database); +std::vector DatabaseCatalog::getDependents(const StorageID & table_id) const +{ + std::lock_guard lock{databases_mutex}; + return referential_dependencies.getDependents(table_id); +} - for (const auto & table : dependent) - { - [[maybe_unused]] bool removed = loading_dependencies[table].dependencies.erase(removing_table); - assert(removed); - } - dependent.clear(); - } - - TableNamesSet dependencies = it->second.dependencies; - for (const auto & table : dependencies) - { - [[maybe_unused]] bool removed = loading_dependencies[table].dependent_database_objects.erase(removing_table); - assert(removed); - } - - loading_dependencies.erase(it); - return dependencies; +std::vector DatabaseCatalog::removeDependencies(const StorageID & table_id, bool check_dependencies, bool is_drop_database) +{ + std::lock_guard lock{databases_mutex}; + if (check_dependencies) + checkTableCanBeRemovedOrRenamedUnlocked(table_id, is_drop_database); + return referential_dependencies.removeDependencies(table_id, /* remove_isolated_tables= */ true); } void DatabaseCatalog::checkTableCanBeRemovedOrRenamed(const StorageID & table_id, bool is_drop_database) const { - QualifiedTableName removing_table = table_id.getQualifiedName(); std::lock_guard lock{databases_mutex}; - auto it = loading_dependencies.find(removing_table); - if (it == loading_dependencies.end()) - return; - - const TableNamesSet & dependent = it->second.dependent_database_objects; - checkTableCanBeRemovedOrRenamedImpl(dependent, removing_table, is_drop_database); + return checkTableCanBeRemovedOrRenamedUnlocked(table_id, is_drop_database); } -void DatabaseCatalog::checkTableCanBeRemovedOrRenamedImpl(const TableNamesSet & dependent, const QualifiedTableName & removing_table, bool is_drop_database) +void DatabaseCatalog::checkTableCanBeRemovedOrRenamedUnlocked(const StorageID & removing_table, bool is_drop_database) const { + const auto & dependents = referential_dependencies.getDependents(removing_table); + if (!is_drop_database) { - if (!dependent.empty()) + if (!dependents.empty()) throw Exception(ErrorCodes::HAVE_DEPENDENT_OBJECTS, "Cannot drop or rename {}, because some tables depend on it: {}", - removing_table, fmt::join(dependent, ", ")); + removing_table, fmt::join(dependents, ", ")); + return; } /// For DROP DATABASE we should ignore dependent tables from the same database. /// TODO unload tables in reverse topological order and remove this code - TableNames from_other_databases; - for (const auto & table : dependent) - if (table.database != removing_table.database) - from_other_databases.push_back(table); + std::vector from_other_databases; + for (const auto & dependent : dependents) + if (dependent.database_name != removing_table.database_name) + from_other_databases.push_back(dependent); if (!from_other_databases.empty()) throw Exception(ErrorCodes::HAVE_DEPENDENT_OBJECTS, "Cannot drop or rename {}, because some tables depend on it: {}", removing_table, fmt::join(from_other_databases, ", ")); } -void DatabaseCatalog::updateLoadingDependencies(const StorageID & table_id, TableNamesSet && new_dependencies) +void DatabaseCatalog::updateDependencies(const StorageID & table_id, const TableNamesSet & new_dependencies) { - if (new_dependencies.empty()) - return; - QualifiedTableName table_name = table_id.getQualifiedName(); std::lock_guard lock{databases_mutex}; - auto it = loading_dependencies.find(table_name); - if (it == loading_dependencies.end()) - it = loading_dependencies.emplace(table_name, DependenciesInfo{}).first; - - auto & old_dependencies = it->second.dependencies; - for (const auto & dependency : old_dependencies) - if (!new_dependencies.contains(dependency)) - loading_dependencies[dependency].dependent_database_objects.erase(table_name); - - for (const auto & dependency : new_dependencies) - if (!old_dependencies.contains(dependency)) - loading_dependencies[dependency].dependent_database_objects.insert(table_name); - - old_dependencies = std::move(new_dependencies); + referential_dependencies.removeDependencies(table_id, /* remove_isolated_tables= */ true); + referential_dependencies.addDependencies(table_id, new_dependencies); } void DatabaseCatalog::cleanupStoreDirectoryTask() @@ -1215,6 +1168,8 @@ void DatabaseCatalog::cleanupStoreDirectoryTask() if (affected_dirs) LOG_INFO(log, "Cleaned up {} directories from store/ on disk {}", affected_dirs, disk_name); + else + LOG_TEST(log, "Nothing to clean up from store/ on disk {}", disk_name); } (*cleanup_task)->scheduleAfter(unused_dir_cleanup_period_sec * 1000); diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index a44099b9fdc..a3fa4515a69 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include @@ -37,11 +37,7 @@ using DatabasePtr = std::shared_ptr; using DatabaseAndTable = std::pair; using Databases = std::map>; using DiskPtr = std::shared_ptr; - -/// Table -> set of table-views that make SELECT from it. -using ViewDependencies = std::map>; -using Dependencies = std::vector; - +using TableNamesSet = std::unordered_set; /// Allows executing DDL query only in one thread. /// Puts an element into the map, locks tables's mutex, counts how much threads run parallel query on the table, @@ -188,12 +184,11 @@ public: /// Four views (tables, views, columns, schemata) in the "information_schema" database are predefined too. bool isPredefinedTable(const StorageID & table_id) const; - void addDependency(const StorageID & from, const StorageID & where); - void removeDependency(const StorageID & from, const StorageID & where); - Dependencies getDependencies(const StorageID & from) const; - - /// For Materialized and Live View - void updateDependency(const StorageID & old_from, const StorageID & old_where,const StorageID & new_from, const StorageID & new_where); + /// View dependencies between a source table and its view. + void addViewDependency(const StorageID & source_table_id, const StorageID & view_id); + void removeViewDependency(const StorageID & source_table_id, const StorageID & view_id); + std::vector getDependentViews(const StorageID & source_table_id) const; + void updateViewDependency(const StorageID & old_source_table_id, const StorageID & old_view_id, const StorageID & new_source_table_id, const StorageID & new_view_id); /// If table has UUID, addUUIDMapping(...) must be called when table attached to some database /// removeUUIDMapping(...) must be called when it detached, @@ -223,16 +218,20 @@ public: void waitTableFinallyDropped(const UUID & uuid); - void addLoadingDependencies(const QualifiedTableName & table, TableNamesSet && dependencies); - void addLoadingDependencies(const DependenciesInfos & new_infos); - DependenciesInfo getLoadingDependenciesInfo(const StorageID & table_id) const; + /// Referential dependencies between tables: table "A" depends on table "B" + /// if "B" is referenced in the definition of "A". + void addDependencies(const StorageID & table_id, const std::vector & dependencies); + void addDependencies(const QualifiedTableName & table_name, const TableNamesSet & dependencies); + void addDependencies(const TablesDependencyGraph & extra_graph); + std::vector removeDependencies(const StorageID & table_id, bool check_dependencies, bool is_drop_database = false); + + std::vector getDependencies(const StorageID & table_id) const; + std::vector getDependents(const StorageID & table_id) const; + + void updateDependencies(const StorageID & table_id, const TableNamesSet & new_dependencies); - TableNamesSet tryRemoveLoadingDependencies(const StorageID & table_id, bool check_dependencies, bool is_drop_database = false); - TableNamesSet tryRemoveLoadingDependenciesUnlocked(const QualifiedTableName & removing_table, bool check_dependencies, bool is_drop_database = false) TSA_REQUIRES(databases_mutex); void checkTableCanBeRemovedOrRenamed(const StorageID & table_id, bool is_drop_database = false) const; - void updateLoadingDependencies(const StorageID & table_id, TableNamesSet && new_dependencies); - private: // The global instance of database catalog. unique_ptr is to allow // deferred initialization. Thought I'd use std::optional, but I can't @@ -245,7 +244,7 @@ private: void shutdownImpl(); - static void checkTableCanBeRemovedOrRenamedImpl(const TableNamesSet & dependent, const QualifiedTableName & removing_table, bool is_drop_database); + void checkTableCanBeRemovedOrRenamedUnlocked(const StorageID & removing_table, bool is_drop_database) const TSA_REQUIRES(databases_mutex); struct UUIDToStorageMapPart { @@ -281,12 +280,15 @@ private: mutable std::mutex databases_mutex; - ViewDependencies view_dependencies TSA_GUARDED_BY(databases_mutex); - Databases databases TSA_GUARDED_BY(databases_mutex); UUIDToStorageMap uuid_map; - DependenciesInfos loading_dependencies TSA_GUARDED_BY(databases_mutex); + /// Referential dependencies between tables: table "A" depends on table "B" + /// if the table "B" is referenced in the definition of the table "A". + TablesDependencyGraph referential_dependencies TSA_GUARDED_BY(databases_mutex); + + /// View dependencies between a source table and its view. + TablesDependencyGraph view_dependencies TSA_GUARDED_BY(databases_mutex); Poco::Logger * log; diff --git a/src/Interpreters/DirectJoin.h b/src/Interpreters/DirectJoin.h index 6a6f4505474..bdbd155dc36 100644 --- a/src/Interpreters/DirectJoin.h +++ b/src/Interpreters/DirectJoin.h @@ -48,7 +48,7 @@ public: virtual bool isFilled() const override { return true; } - virtual std::shared_ptr + virtual IBlocksStreamPtr getNonJoinedBlocks(const Block &, const Block &, UInt64) const override { return nullptr; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index b34dbf3128c..17788fce53f 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -1009,12 +1010,26 @@ static ActionsDAGPtr createJoinedBlockActions(ContextPtr context, const TableJoi std::shared_ptr tryKeyValueJoin(std::shared_ptr analyzed_join, const Block & right_sample_block); -static std::shared_ptr chooseJoinAlgorithm(std::shared_ptr analyzed_join, std::unique_ptr & joined_plan, ContextPtr context) + +static std::shared_ptr chooseJoinAlgorithm( + std::shared_ptr analyzed_join, const ColumnsWithTypeAndName & left_sample_columns, std::unique_ptr & joined_plan, ContextPtr context) { + const auto & settings = context->getSettings(); + + Block left_sample_block(left_sample_columns); + for (auto & column : left_sample_block) + { + if (!column.column) + column.column = column.type->createColumn(); + } + Block right_sample_block = joined_plan->getCurrentDataStream().header; + std::vector tried_algorithms; + if (analyzed_join->isEnabledAlgorithm(JoinAlgorithm::DIRECT)) { + tried_algorithms.push_back(toString(JoinAlgorithm::DIRECT)); JoinPtr direct_join = tryKeyValueJoin(analyzed_join, right_sample_block); if (direct_join) { @@ -1027,6 +1042,7 @@ static std::shared_ptr chooseJoinAlgorithm(std::shared_ptr ana if (analyzed_join->isEnabledAlgorithm(JoinAlgorithm::PARTIAL_MERGE) || analyzed_join->isEnabledAlgorithm(JoinAlgorithm::PREFER_PARTIAL_MERGE)) { + tried_algorithms.push_back(toString(JoinAlgorithm::PARTIAL_MERGE)); if (MergeJoin::isSupported(analyzed_join)) return std::make_shared(analyzed_join, right_sample_block); } @@ -1036,22 +1052,37 @@ static std::shared_ptr chooseJoinAlgorithm(std::shared_ptr ana analyzed_join->isEnabledAlgorithm(JoinAlgorithm::PREFER_PARTIAL_MERGE) || analyzed_join->isEnabledAlgorithm(JoinAlgorithm::PARALLEL_HASH)) { + tried_algorithms.push_back(toString(JoinAlgorithm::HASH)); if (analyzed_join->allowParallelHashJoin()) - return std::make_shared(context, analyzed_join, context->getSettings().max_threads, right_sample_block); + return std::make_shared(context, analyzed_join, settings.max_threads, right_sample_block); return std::make_shared(analyzed_join, right_sample_block); } if (analyzed_join->isEnabledAlgorithm(JoinAlgorithm::FULL_SORTING_MERGE)) { + tried_algorithms.push_back(toString(JoinAlgorithm::FULL_SORTING_MERGE)); if (FullSortingMergeJoin::isSupported(analyzed_join)) return std::make_shared(analyzed_join, right_sample_block); } - if (analyzed_join->isEnabledAlgorithm(JoinAlgorithm::AUTO)) - return std::make_shared(analyzed_join, right_sample_block); + if (analyzed_join->isEnabledAlgorithm(JoinAlgorithm::GRACE_HASH)) + { + tried_algorithms.push_back(toString(JoinAlgorithm::GRACE_HASH)); + if (GraceHashJoin::isSupported(analyzed_join)) + return std::make_shared(context, analyzed_join, left_sample_block, right_sample_block, context->getTempDataOnDisk()); + } - throw Exception("Can't execute any of specified algorithms for specified strictness/kind and right storage type", - ErrorCodes::NOT_IMPLEMENTED); + if (analyzed_join->isEnabledAlgorithm(JoinAlgorithm::AUTO)) + { + tried_algorithms.push_back(toString(JoinAlgorithm::AUTO)); + + if (MergeJoin::isSupported(analyzed_join)) + return std::make_shared(analyzed_join, right_sample_block); + } + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Can't execute {} join algorithm for this strictness/kind and right storage type", + fmt::join(tried_algorithms, " or ")); } static std::unique_ptr buildJoinedPlan( @@ -1186,7 +1217,7 @@ JoinPtr SelectQueryExpressionAnalyzer::makeJoin( joined_plan->addStep(std::move(converting_step)); } - JoinPtr join = chooseJoinAlgorithm(analyzed_join, joined_plan, getContext()); + JoinPtr join = chooseJoinAlgorithm(analyzed_join, left_columns, joined_plan, getContext()); return join; } diff --git a/src/Interpreters/FullSortingMergeJoin.h b/src/Interpreters/FullSortingMergeJoin.h index 14c81259159..fa7d0478535 100644 --- a/src/Interpreters/FullSortingMergeJoin.h +++ b/src/Interpreters/FullSortingMergeJoin.h @@ -100,7 +100,7 @@ public: bool alwaysReturnsEmptySet() const override { return false; } - std::shared_ptr + IBlocksStreamPtr getNonJoinedBlocks(const Block & /* left_sample_block */, const Block & /* result_sample_block */, UInt64 /* max_block_size */) const override { throw Exception(ErrorCodes::LOGICAL_ERROR, "FullSortingMergeJoin::getNonJoinedBlocks should not be called"); diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp new file mode 100644 index 00000000000..5ef27613591 --- /dev/null +++ b/src/Interpreters/GraceHashJoin.cpp @@ -0,0 +1,628 @@ +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace CurrentMetrics +{ + extern const Metric TemporaryFilesForJoin; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LIMIT_EXCEEDED; + extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; +} + +namespace +{ + class AccumulatedBlockReader + { + public: + AccumulatedBlockReader(TemporaryFileStream & reader_, + std::mutex & mutex_, + size_t result_block_size_ = DEFAULT_BLOCK_SIZE * 8) + : reader(reader_) + , mutex(mutex_) + , result_block_size(result_block_size_) + { + if (!reader.isWriteFinished()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Reading not finished file"); + } + + Block read() + { + std::lock_guard lock(mutex); + + if (eof) + return {}; + + Blocks blocks; + size_t rows_read = 0; + while (rows_read < result_block_size) + { + Block block = reader.read(); + rows_read += block.rows(); + if (!block) + { + eof = true; + return concatenateBlocks(blocks); + } + blocks.push_back(std::move(block)); + } + + return concatenateBlocks(blocks); + } + + private: + TemporaryFileStream & reader; + std::mutex & mutex; + + const size_t result_block_size; + bool eof = false; + }; + + std::deque generateRandomPermutation(size_t from, size_t to) + { + size_t size = to - from; + std::deque indices(size); + std::iota(indices.begin(), indices.end(), from); + std::shuffle(indices.begin(), indices.end(), thread_local_rng); + return indices; + } + + // Try to apply @callback in the order specified in @indices + // Until it returns true for each index in the @indices. + void retryForEach(std::deque indices, Fn auto callback) + { + while (!indices.empty()) + { + size_t bucket_index = indices.front(); + indices.pop_front(); + + if (!callback(bucket_index)) + indices.push_back(bucket_index); + } + } +} + +class GraceHashJoin::FileBucket : boost::noncopyable +{ + enum class State : int + { + WRITING_BLOCKS, + JOINING_BLOCKS, + FINISHED, + }; + +public: + using BucketLock = std::unique_lock; + + struct Stats + { + TemporaryFileStream::Stat left; + TemporaryFileStream::Stat right; + }; + + explicit FileBucket(size_t bucket_index_, + TemporaryFileStream & left_file_, + TemporaryFileStream & right_file_, + Poco::Logger * log_) + : idx{bucket_index_} + , left_file{left_file_} + , right_file{right_file_} + , state{State::WRITING_BLOCKS} + , log(log_) + { + } + + void addLeftBlock(const Block & block) + { + std::unique_lock lock(left_file_mutex); + addBlockImpl(block, left_file, lock); + } + + void addRightBlock(const Block & block) + { + std::unique_lock lock(right_file_mutex); + addBlockImpl(block, right_file, lock); + } + + bool tryAddLeftBlock(const Block & block) + { + std::unique_lock lock(left_file_mutex, std::try_to_lock); + return addBlockImpl(block, left_file, lock); + } + + bool tryAddRightBlock(const Block & block) + { + std::unique_lock lock(right_file_mutex, std::try_to_lock); + return addBlockImpl(block, right_file, lock); + } + + bool finished() const + { + std::unique_lock left_lock(left_file_mutex); + return left_file.isEof(); + } + + bool empty() const { return is_empty.load(); } + + Stats getStat() const { return stats; } + + AccumulatedBlockReader startJoining() + { + LOG_TRACE(log, "Joining file bucket {}", idx); + + { + std::unique_lock left_lock(left_file_mutex); + std::unique_lock right_lock(right_file_mutex); + + stats.left = left_file.finishWriting(); + stats.right = right_file.finishWriting(); + state = State::JOINING_BLOCKS; + } + + return AccumulatedBlockReader(right_file, right_file_mutex); + } + + AccumulatedBlockReader getLeftTableReader() + { + ensureState(State::JOINING_BLOCKS); + return AccumulatedBlockReader(left_file, left_file_mutex); + } + + const size_t idx; + +private: + bool addBlockImpl(const Block & block, TemporaryFileStream & writer, std::unique_lock & lock) + { + ensureState(State::WRITING_BLOCKS); + + if (!lock.owns_lock()) + return false; + + if (block.rows()) + is_empty = false; + + writer.write(block); + return true; + } + + void transition(State expected, State desired) + { + State prev = state.exchange(desired); + if (prev != expected) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid state transition from {} (got {}) to {}", expected, prev, desired); + } + + void ensureState(State expected) const + { + State cur_state = state.load(); + if (cur_state != expected) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid state transition, expected {}, got {}", expected, state.load()); + } + + TemporaryFileStream & left_file; + TemporaryFileStream & right_file; + mutable std::mutex left_file_mutex; + mutable std::mutex right_file_mutex; + + std::atomic_bool is_empty = true; + + std::atomic state; + Stats stats; + + Poco::Logger * log; +}; + + +static void flushBlocksToBuckets(Blocks & blocks, const GraceHashJoin::Buckets & buckets_snapshot) +{ + assert(blocks.size() == buckets_snapshot.size()); + retryForEach( + generateRandomPermutation(1, buckets_snapshot.size()), + [&](size_t i) + { + if (!blocks[i].rows()) + return true; + bool flushed = buckets_snapshot[i]->tryAddRightBlock(blocks[i]); + if (flushed) + blocks[i].clear(); + return flushed; + }); +} + +GraceHashJoin::GraceHashJoin( + ContextPtr context_, std::shared_ptr table_join_, + const Block & left_sample_block_, + const Block & right_sample_block_, + TemporaryDataOnDiskScopePtr tmp_data_, + bool any_take_last_row_) + : log{&Poco::Logger::get("GraceHashJoin")} + , context{context_} + , table_join{std::move(table_join_)} + , left_sample_block{left_sample_block_} + , right_sample_block{right_sample_block_} + , any_take_last_row{any_take_last_row_} + , max_num_buckets{context->getSettingsRef().grace_hash_join_max_buckets} + , max_block_size{context->getSettingsRef().max_block_size} + , left_key_names(table_join->getOnlyClause().key_names_left) + , right_key_names(table_join->getOnlyClause().key_names_right) + , tmp_data(std::make_unique(tmp_data_, CurrentMetrics::TemporaryFilesForJoin)) + , hash_join(makeInMemoryJoin()) +{ + if (!GraceHashJoin::isSupported(table_join)) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "GraceHashJoin is not supported for this join type"); + +} + +void GraceHashJoin::initBuckets() +{ + const auto & settings = context->getSettingsRef(); + + size_t initial_num_buckets = roundUpToPowerOfTwoOrZero(std::clamp(settings.grace_hash_join_initial_buckets, 1, settings.grace_hash_join_max_buckets)); + + for (size_t i = 0; i < initial_num_buckets; ++i) + { + addBucket(buckets); + } + + if (buckets.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "No buckets created"); + + LOG_TRACE(log, "Initialize {} buckets", buckets.size()); + + current_bucket = buckets.front().get(); + current_bucket->startJoining(); +} + +bool GraceHashJoin::isSupported(const std::shared_ptr & table_join) +{ + bool is_asof = (table_join->strictness() == JoinStrictness::Asof); + return !is_asof && isInnerOrLeft(table_join->kind()) && table_join->oneDisjunct(); +} + +GraceHashJoin::~GraceHashJoin() = default; + +bool GraceHashJoin::addJoinedBlock(const Block & block, bool /*check_limits*/) +{ + if (current_bucket == nullptr) + throw Exception(ErrorCodes::LOGICAL_ERROR, "GraceHashJoin is not initialized"); + + Block materialized = materializeBlock(block); + addJoinedBlockImpl(materialized); + return true; +} + +bool GraceHashJoin::fitsInMemory() const +{ + /// One row can't be split, avoid loop + if (hash_join->getTotalRowCount() < 2) + return true; + + return table_join->sizeLimits().softCheck(hash_join->getTotalRowCount(), hash_join->getTotalByteCount()); +} + +GraceHashJoin::Buckets GraceHashJoin::rehashBuckets(size_t to_size) +{ + std::unique_lock lock(rehash_mutex); + size_t current_size = buckets.size(); + + if (to_size <= current_size) + return buckets; + + assert(isPowerOf2(to_size)); + + if (to_size > max_num_buckets) + { + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "Too many grace hash join buckets ({} > {}), consider increasing grace_hash_join_max_buckets or max_rows_in_join/max_bytes_in_join", + to_size, max_num_buckets); + } + + LOG_TRACE(log, "Rehashing from {} to {}", current_size, to_size); + + buckets.reserve(to_size); + for (size_t i = current_size; i < to_size; ++i) + addBucket(buckets); + + return buckets; +} + +void GraceHashJoin::addBucket(Buckets & destination) +{ + BucketPtr new_bucket = std::make_shared( + destination.size(), tmp_data->createStream(left_sample_block), tmp_data->createStream(right_sample_block), log); + destination.emplace_back(std::move(new_bucket)); +} + +void GraceHashJoin::checkTypesOfKeys(const Block & block) const +{ + assert(hash_join); + return hash_join->checkTypesOfKeys(block); +} + +void GraceHashJoin::initialize(const Block & sample_block) +{ + left_sample_block = sample_block.cloneEmpty(); + output_sample_block = left_sample_block.cloneEmpty(); + ExtraBlockPtr not_processed; + hash_join->joinBlock(output_sample_block, not_processed); + initBuckets(); +} + +void GraceHashJoin::joinBlock(Block & block, std::shared_ptr & not_processed) +{ + if (block.rows() == 0) + { + hash_join->joinBlock(block, not_processed); + return; + } + + materializeBlockInplace(block); + + Buckets buckets_snapshot = getCurrentBuckets(); + size_t num_buckets = buckets_snapshot.size(); + Blocks blocks = JoinCommon::scatterBlockByHash(left_key_names, block, num_buckets); + + block = std::move(blocks[current_bucket->idx]); + + hash_join->joinBlock(block, not_processed); + if (not_processed) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unhandled not processed block in GraceHashJoin"); + + // We need to skip the first bucket that is already joined in memory, so we start with 1. + retryForEach( + generateRandomPermutation(1, num_buckets), + [&blocks, &buckets_snapshot](size_t idx) + { + if (blocks[idx].rows() == 0) + return true; + return buckets_snapshot[idx]->tryAddLeftBlock(blocks[idx]); + }); +} + +void GraceHashJoin::setTotals(const Block & block) +{ + if (block.rows() > 0) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Totals are not supported for GraceHashJoin, got '{}'", block.dumpStructure()); +} + +size_t GraceHashJoin::getTotalRowCount() const +{ + std::lock_guard lock(hash_join_mutex); + assert(hash_join); + return hash_join->getTotalRowCount(); +} + +size_t GraceHashJoin::getTotalByteCount() const +{ + std::lock_guard lock(hash_join_mutex); + assert(hash_join); + return hash_join->getTotalByteCount(); +} + +bool GraceHashJoin::alwaysReturnsEmptySet() const +{ + if (!isInnerOrRight(table_join->kind())) + return false; + + std::shared_lock lock(rehash_mutex); + + bool file_buckets_are_empty = std::all_of(buckets.begin(), buckets.end(), [](const auto & bucket) { return bucket->empty(); }); + bool hash_join_is_empty = hash_join && hash_join->alwaysReturnsEmptySet(); + + return hash_join_is_empty && file_buckets_are_empty; +} + +IBlocksStreamPtr GraceHashJoin::getNonJoinedBlocks(const Block &, const Block &, UInt64) const +{ + /// We do no support returning non joined blocks here. + /// TODO: They _should_ be reported by getDelayedBlocks instead + return nullptr; +} + +class GraceHashJoin::DelayedBlocks : public IBlocksStream +{ +public: + explicit DelayedBlocks(size_t current_bucket_, Buckets buckets_, InMemoryJoinPtr hash_join_, const Names & left_key_names_, const Names & right_key_names_) + : current_bucket(current_bucket_) + , buckets(std::move(buckets_)) + , hash_join(std::move(hash_join_)) + , left_reader(buckets[current_bucket]->getLeftTableReader()) + , left_key_names(left_key_names_) + , right_key_names(right_key_names_) + { + } + + Block nextImpl() override + { + Block block; + size_t num_buckets = buckets.size(); + size_t current_idx = buckets[current_bucket]->idx; + + do + { + block = left_reader.read(); + if (!block) + { + return {}; + } + + Blocks blocks = JoinCommon::scatterBlockByHash(left_key_names, block, num_buckets); + block = std::move(blocks[current_idx]); + + /* + * We need to filter out blocks that were written to the current bucket `B_{n}` + * but then virtually moved to another bucket `B_{n+i}` on rehash. + * Bucket `B_{n+i}` is waiting for the buckets with smaller index to be processed, + * and rows can be moved only forward (because we increase hash modulo twice on each rehash), + * so it is safe to add blocks. + */ + for (size_t bucket_idx = 0; bucket_idx < num_buckets; ++bucket_idx) + { + if (blocks[bucket_idx].rows() == 0) + continue; + + if (bucket_idx == current_idx) // Rows that are still in our bucket + continue; + + buckets[bucket_idx]->addLeftBlock(blocks[bucket_idx]); + } + } while (block.rows() == 0); + + ExtraBlockPtr not_processed; + hash_join->joinBlock(block, not_processed); + + if (not_processed) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unsupported hash join type"); + + return block; + } + + size_t current_bucket; + Buckets buckets; + InMemoryJoinPtr hash_join; + + AccumulatedBlockReader left_reader; + + Names left_key_names; + Names right_key_names; +}; + +IBlocksStreamPtr GraceHashJoin::getDelayedBlocks() +{ + std::lock_guard current_bucket_lock(current_bucket_mutex); + + if (current_bucket == nullptr) + return nullptr; + + size_t bucket_idx = current_bucket->idx; + + if (hash_join) + { + auto right_blocks = hash_join->releaseJoinedBlocks(); + Blocks blocks = JoinCommon::scatterBlockByHash(right_key_names, right_blocks, buckets.size()); + + for (size_t i = 0; i < blocks.size(); ++i) + { + if (blocks[i].rows() == 0 || i == bucket_idx) + continue; + + if (i < bucket_idx) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected bucket index {} when current bucket is {}", i, bucket_idx); + buckets[i]->addRightBlock(blocks[i]); + } + } + + hash_join = makeInMemoryJoin(); + + for (bucket_idx = bucket_idx + 1; bucket_idx < buckets.size(); ++bucket_idx) + { + current_bucket = buckets[bucket_idx].get(); + if (current_bucket->finished() || current_bucket->empty()) + { + LOG_TRACE(log, "Skipping {} {} bucket {}", + current_bucket->finished() ? "finished" : "", + current_bucket->empty() ? "empty" : "", + bucket_idx); + continue; + } + + auto right_reader = current_bucket->startJoining(); + size_t num_rows = 0; /// count rows that were written and rehashed + while (Block block = right_reader.read()) + { + num_rows += block.rows(); + addJoinedBlockImpl(std::move(block)); + } + + LOG_TRACE(log, "Loaded bucket {} with {}(/{}) rows", + bucket_idx, hash_join->getTotalRowCount(), num_rows); + + return std::make_unique(current_bucket->idx, buckets, hash_join, left_key_names, right_key_names); + } + + LOG_TRACE(log, "Finished loading all buckets"); + + current_bucket = nullptr; + return nullptr; +} + +GraceHashJoin::InMemoryJoinPtr GraceHashJoin::makeInMemoryJoin() +{ + return std::make_unique(table_join, right_sample_block, any_take_last_row); +} + +void GraceHashJoin::addJoinedBlockImpl(Block block) +{ + Buckets buckets_snapshot = getCurrentBuckets(); + Blocks blocks = JoinCommon::scatterBlockByHash(right_key_names, block, buckets_snapshot.size()); + size_t bucket_index = current_bucket->idx; + + // Add block to the in-memory join + if (blocks[bucket_index].rows() > 0) + { + std::lock_guard lock(hash_join_mutex); + + hash_join->addJoinedBlock(blocks[bucket_index], /* check_limits = */ false); + bool overflow = !fitsInMemory(); + + if (overflow) + { + auto right_blocks = hash_join->releaseJoinedBlocks(); + right_blocks.pop_back(); + + for (const auto & right_block : right_blocks) + blocks.push_back(right_block); + } + + while (overflow) + { + buckets_snapshot = rehashBuckets(buckets_snapshot.size() * 2); + + blocks = JoinCommon::scatterBlockByHash(right_key_names, blocks, buckets_snapshot.size()); + hash_join = makeInMemoryJoin(); + hash_join->addJoinedBlock(blocks[bucket_index], /* check_limits = */ false); + overflow = !fitsInMemory(); + } + blocks[bucket_index].clear(); + } + + flushBlocksToBuckets(blocks, buckets_snapshot); +} + +size_t GraceHashJoin::getNumBuckets() const +{ + std::shared_lock lock(rehash_mutex); + return buckets.size(); +} + +GraceHashJoin::Buckets GraceHashJoin::getCurrentBuckets() const +{ + std::shared_lock lock(rehash_mutex); + return buckets; +} + +} diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h new file mode 100644 index 00000000000..f4e75f142f3 --- /dev/null +++ b/src/Interpreters/GraceHashJoin.h @@ -0,0 +1,142 @@ +#pragma once + +#include +#include +#include + +#include + +#include + +#include + +namespace DB +{ + +class TableJoin; +class HashJoin; + +/** + * Efficient and highly parallel implementation of external memory JOIN based on HashJoin. + * Supports most of the JOIN modes, except CROSS and ASOF. + * + * The joining algorithm consists of three stages: + * + * 1) During the first stage we accumulate blocks of the right table via @addJoinedBlock. + * Each input block is split into multiple buckets based on the hash of the row join keys. + * The first bucket is added to the in-memory HashJoin, and the remaining buckets are written to disk for further processing. + * When the size of HashJoin exceeds the limits, we double the number of buckets. + * There can be multiple threads calling addJoinedBlock, just like @ConcurrentHashJoin. + * + * 2) At the second stage we process left table blocks via @joinBlock. + * Again, each input block is split into multiple buckets by hash. + * The first bucket is joined in-memory via HashJoin::joinBlock, and the remaining buckets are written to the disk. + * + * 3) When the last thread reading left table block finishes, the last stage begins. + * Each @DelayedJoinedBlocksTransform calls repeatedly @getDelayedBlocks until there are no more unfinished buckets left. + * Inside @getDelayedBlocks we select the next unprocessed bucket, load right table blocks from disk into in-memory HashJoin, + * And then join them with left table blocks. + * + * After joining the left table blocks, we can load non-joined rows from the right table for RIGHT/FULL JOINs. + * Note that non-joined rows are processed in multiple threads, unlike HashJoin/ConcurrentHashJoin/MergeJoin. + */ +class GraceHashJoin final : public IJoin +{ + class FileBucket; + class DelayedBlocks; + using InMemoryJoin = HashJoin; + + using InMemoryJoinPtr = std::shared_ptr; + +public: + using BucketPtr = std::shared_ptr; + using Buckets = std::vector; + + GraceHashJoin( + ContextPtr context_, std::shared_ptr table_join_, + const Block & left_sample_block_, const Block & right_sample_block_, + TemporaryDataOnDiskScopePtr tmp_data_, + bool any_take_last_row_ = false); + + ~GraceHashJoin() override; + + const TableJoin & getTableJoin() const override { return *table_join; } + + void initialize(const Block & sample_block) override; + + bool addJoinedBlock(const Block & block, bool check_limits) override; + void checkTypesOfKeys(const Block & block) const override; + void joinBlock(Block & block, std::shared_ptr & not_processed) override; + + void setTotals(const Block & block) override; + + size_t getTotalRowCount() const override; + size_t getTotalByteCount() const override; + bool alwaysReturnsEmptySet() const override; + + bool supportParallelJoin() const override { return true; } + bool supportTotals() const override { return false; } + + IBlocksStreamPtr + getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const override; + + /// Open iterator over joined blocks. + /// Must be called after all @joinBlock calls. + IBlocksStreamPtr getDelayedBlocks() override; + bool hasDelayedBlocks() const override { return true; } + + static bool isSupported(const std::shared_ptr & table_join); + +private: + void initBuckets(); + /// Create empty join for in-memory processing. + InMemoryJoinPtr makeInMemoryJoin(); + + /// Add right table block to the @join. Calls @rehash on overflow. + void addJoinedBlockImpl(Block block); + + /// Check that @join satisifes limits on rows/bytes in @table_join. + bool fitsInMemory() const; + + /// Create new bucket at the end of @destination. + void addBucket(Buckets & destination); + + /// Increase number of buckets to match desired_size. + /// Called when HashJoin in-memory table for one bucket exceeds the limits. + /// + /// NB: after @rehashBuckets there may be rows that are written to the buckets that they do not belong to. + /// It is fine; these rows will be written to the corresponding buckets during the third stage. + Buckets rehashBuckets(size_t to_size); + + /// Perform some bookkeeping after all calls to @joinBlock. + void startReadingDelayedBlocks(); + + size_t getNumBuckets() const; + Buckets getCurrentBuckets() const; + + Poco::Logger * log; + ContextPtr context; + std::shared_ptr table_join; + Block left_sample_block; + Block right_sample_block; + Block output_sample_block; + bool any_take_last_row; + const size_t max_num_buckets; + size_t max_block_size; + + Names left_key_names; + Names right_key_names; + + TemporaryDataOnDiskPtr tmp_data; + + Buckets buckets; + mutable std::shared_mutex rehash_mutex; + + FileBucket * current_bucket = nullptr; + mutable std::mutex current_bucket_mutex; + + InMemoryJoinPtr hash_join; + mutable std::mutex hash_join_mutex; +}; + +} diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 41c7c28a6fa..6f1634b4e39 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -225,7 +226,6 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s , log(&Poco::Logger::get("HashJoin")) { LOG_DEBUG(log, "HashJoin. Datatype: {}, kind: {}, strictness: {}", data->type, kind, strictness); - LOG_DEBUG(log, "Right sample block: {}", right_sample_block.dumpStructure()); if (isCrossOrComma(kind)) { @@ -249,15 +249,6 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s sample_block_with_columns_to_add = right_table_keys = materializeBlock(right_sample_block); } - LOG_TRACE(log, "Columns to add: [{}], required right [{}]", - sample_block_with_columns_to_add.dumpStructure(), fmt::join(required_right_keys.getNames(), ", ")); - { - std::vector log_text; - for (const auto & clause : table_join->getClauses()) - log_text.push_back(clause.formatDebug()); - LOG_TRACE(log, "Joining on: {}", fmt::join(log_text, " | ")); - } - JoinCommon::convertToFullColumnsInplace(right_table_keys); initRightBlockStructure(data->sample_block); @@ -644,7 +635,10 @@ void HashJoin::initRightBlockStructure(Block & saved_block_sample) bool multiple_disjuncts = !table_join->oneDisjunct(); /// We could remove key columns for LEFT | INNER HashJoin but we should keep them for JoinSwitcher (if any). - bool save_key_columns = table_join->isEnabledAlgorithm(JoinAlgorithm::AUTO) || isRightOrFull(kind) || multiple_disjuncts; + bool save_key_columns = table_join->isEnabledAlgorithm(JoinAlgorithm::AUTO) || + table_join->isEnabledAlgorithm(JoinAlgorithm::GRACE_HASH) || + isRightOrFull(kind) || + multiple_disjuncts; if (save_key_columns) { saved_block_sample = right_table_keys.cloneEmpty(); @@ -887,7 +881,8 @@ public: static void assertBlockEqualsStructureUpToLowCard(const Block & lhs_block, const Block & rhs_block) { if (lhs_block.columns() != rhs_block.columns()) - throw Exception("Different number of columns in blocks", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Different number of columns in blocks [{}] and [{}]", + lhs_block.dumpStructure(), rhs_block.dumpStructure()); for (size_t i = 0; i < lhs_block.columns(); ++i) { @@ -1684,6 +1679,9 @@ void HashJoin::checkTypesOfKeys(const Block & block) const void HashJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) { + if (data->released) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot join after data has been released"); + for (const auto & onexpr : table_join->getClauses()) { auto cond_column_name = onexpr.condColumnNames(); @@ -1951,16 +1949,13 @@ private: } }; -std::shared_ptr HashJoin::getNonJoinedBlocks(const Block & left_sample_block, +IBlocksStreamPtr HashJoin::getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const { - if (table_join->strictness() == JoinStrictness::Asof || - table_join->strictness() == JoinStrictness::Semi || - !isRightOrFull(table_join->kind())) - { + if (!JoinCommon::hasNonJoinedBlocks(*table_join)) return {}; - } + bool multiple_disjuncts = !table_join->oneDisjunct(); if (multiple_disjuncts) @@ -1968,7 +1963,7 @@ std::shared_ptr HashJoin::getNonJoinedBlocks(const Block & left /// ... calculate `left_columns_count` ... size_t left_columns_count = left_sample_block.columns(); auto non_joined = std::make_unique>(*this, max_block_size); - return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); + return std::make_unique(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); } else @@ -1976,7 +1971,7 @@ std::shared_ptr HashJoin::getNonJoinedBlocks(const Block & left size_t left_columns_count = left_sample_block.columns(); assert(left_columns_count == result_sample_block.columns() - required_right_keys.columns() - sample_block_with_columns_to_add.columns()); auto non_joined = std::make_unique>(*this, max_block_size); - return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); + return std::make_unique(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); } } @@ -1998,6 +1993,41 @@ void HashJoin::reuseJoinedData(const HashJoin & join) } } +BlocksList HashJoin::releaseJoinedBlocks() +{ + BlocksList right_blocks = std::move(data->blocks); + data->released = true; + BlocksList restored_blocks; + + /// names to positions optimization + std::vector positions; + std::vector is_nullable; + if (!right_blocks.empty()) + { + positions.reserve(right_sample_block.columns()); + const Block & tmp_block = *right_blocks.begin(); + for (const auto & sample_column : right_sample_block) + { + positions.emplace_back(tmp_block.getPositionByName(sample_column.name)); + is_nullable.emplace_back(JoinCommon::isNullable(sample_column.type)); + } + } + + for (Block & saved_block : right_blocks) + { + Block restored_block; + for (size_t i = 0; i < positions.size(); ++i) + { + auto & column = saved_block.getByPosition(positions[i]); + restored_block.insert(correctNullability(std::move(column), is_nullable[i])); + } + restored_blocks.emplace_back(std::move(restored_block)); + } + + return restored_blocks; +} + + const ColumnWithTypeAndName & HashJoin::rightAsofKeyColumn() const { /// It should be nullable when right side is nullable diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 587fed9b4a6..5ea47823b69 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -187,7 +186,7 @@ public: * Use only after all calls to joinBlock was done. * left_sample_block is passed without account of 'use_nulls' setting (columns will be converted to Nullable inside). */ - std::shared_ptr getNonJoinedBlocks( + IBlocksStreamPtr getNonJoinedBlocks( const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const override; /// Number of keys in all built JOIN maps. @@ -336,6 +335,8 @@ public: /// Additional data - strings for string keys and continuation elements of single-linked lists of references to rows. Arena pool; + + bool released = false; }; using RightTableDataPtr = std::shared_ptr; @@ -350,10 +351,13 @@ public: void reuseJoinedData(const HashJoin & join); RightTableDataPtr getJoinedData() const { return data; } + BlocksList releaseJoinedBlocks(); bool isUsed(size_t off) const { return used_flags.getUsedSafe(off); } bool isUsed(const Block * block_ptr, size_t row_idx) const { return used_flags.getUsedSafe(block_ptr, row_idx); } + void debugKeys() const; + private: template friend class NotJoinedHash; diff --git a/src/Interpreters/IJoin.h b/src/Interpreters/IJoin.h index b699988e926..69d69ce30a6 100644 --- a/src/Interpreters/IJoin.h +++ b/src/Interpreters/IJoin.h @@ -7,17 +7,21 @@ #include #include #include +#include namespace DB { -class Block; - struct ExtraBlock; using ExtraBlockPtr = std::shared_ptr; class TableJoin; class NotJoinedBlocks; +class IBlocksStream; +using IBlocksStreamPtr = std::shared_ptr; + +class IJoin; +using JoinPtr = std::shared_ptr; enum class JoinPipelineType { @@ -51,6 +55,12 @@ public: /// @returns false, if some limit was exceeded and you should not insert more data. virtual bool addJoinedBlock(const Block & block, bool check_limits = true) = 0; /// NOLINT + /* Some initialization may be required before joinBlock() call. + * It's better to done in in constructor, but left block exact structure is not known at that moment. + * TODO: pass correct left block sample to the constructor. + */ + virtual void initialize(const Block & /* left_sample_block */) {} + virtual void checkTypesOfKeys(const Block & block) const = 0; /// Join the block with data from left hand of JOIN to the right hand data (that was previously built by calls to addJoinedBlock). @@ -77,15 +87,44 @@ public: // That can run FillingRightJoinSideTransform parallelly virtual bool supportParallelJoin() const { return false; } + virtual bool supportTotals() const { return true; } - virtual std::shared_ptr - getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const = 0; + /// Peek next stream of delayed joined blocks. + virtual IBlocksStreamPtr getDelayedBlocks() { return nullptr; } + virtual bool hasDelayedBlocks() const { return false; } + + virtual IBlocksStreamPtr + getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const = 0; private: Block totals; }; +class IBlocksStream +{ +public: + /// Returns empty block on EOF + Block next() + { + if (finished) + return {}; -using JoinPtr = std::shared_ptr; + if (Block res = nextImpl()) + return res; + + finished = true; + return {}; + } + + virtual ~IBlocksStream() = default; + + bool isFinished() const { return finished; } + +protected: + virtual Block nextImpl() = 0; + + std::atomic_bool finished{false}; + +}; } diff --git a/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp b/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp new file mode 100644 index 00000000000..cda91cd4ba1 --- /dev/null +++ b/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp @@ -0,0 +1,28 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +BlockIO InterpreterAlterNamedCollectionQuery::execute() +{ + auto current_context = getContext(); + current_context->checkAccess(AccessType::ALTER_NAMED_COLLECTION); + + const auto & query = query_ptr->as(); + if (!query.cluster.empty()) + { + DDLQueryOnClusterParams params; + return executeDDLQueryOnCluster(query_ptr, current_context, params); + } + + NamedCollectionUtils::updateFromSQL(query, current_context); + return {}; +} + +} diff --git a/src/Interpreters/InterpreterAlterNamedCollectionQuery.h b/src/Interpreters/InterpreterAlterNamedCollectionQuery.h new file mode 100644 index 00000000000..889a41f2cb5 --- /dev/null +++ b/src/Interpreters/InterpreterAlterNamedCollectionQuery.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace DB +{ + +class Context; + +class InterpreterAlterNamedCollectionQuery : public IInterpreter, WithMutableContext +{ +public: + InterpreterAlterNamedCollectionQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) {} + + BlockIO execute() override; + +private: + ASTPtr query_ptr; +}; + +} diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 82f635017c9..22edac051a5 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -22,6 +22,9 @@ #include #include +#include +#include + #include #include @@ -66,6 +69,9 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) { BlockIO res; + if (!UserDefinedSQLFunctionFactory::instance().empty()) + UserDefinedSQLFunctionVisitor::visit(query_ptr); + if (!alter.cluster.empty() && !maybeRemoveOnCluster(query_ptr, getContext())) { DDLQueryOnClusterParams params; diff --git a/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp b/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp new file mode 100644 index 00000000000..c7397d3d64c --- /dev/null +++ b/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp @@ -0,0 +1,30 @@ +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ + +BlockIO InterpreterCreateNamedCollectionQuery::execute() +{ + auto current_context = getContext(); + current_context->checkAccess(AccessType::CREATE_NAMED_COLLECTION); + + const auto & query = query_ptr->as(); + + if (!query.cluster.empty()) + { + DDLQueryOnClusterParams params; + return executeDDLQueryOnCluster(query_ptr, current_context, params); + } + + NamedCollectionUtils::createFromSQL(query, current_context); + return {}; +} + +} diff --git a/src/Interpreters/InterpreterCreateNamedCollectionQuery.h b/src/Interpreters/InterpreterCreateNamedCollectionQuery.h new file mode 100644 index 00000000000..26335f618ad --- /dev/null +++ b/src/Interpreters/InterpreterCreateNamedCollectionQuery.h @@ -0,0 +1,23 @@ +#pragma once + +#include + + +namespace DB +{ + +class InterpreterCreateNamedCollectionQuery : public IInterpreter, WithMutableContext +{ +public: + InterpreterCreateNamedCollectionQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) + { + } + + BlockIO execute() override; + +private: + ASTPtr query_ptr; +}; + +} diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index e9cf06c5c69..50536b66185 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -58,6 +58,7 @@ #include #include #include +#include #include @@ -71,6 +72,9 @@ #include #include +#include +#include + #define MAX_FIXEDSTRING_SIZE_WITHOUT_SUSPICIOUS 256 @@ -368,6 +372,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns, const char * alias_end = alias_pos + alias.size(); ParserExpression expression_parser; column_declaration->default_expression = parseQuery(expression_parser, alias_pos, alias_end, "expression", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + column_declaration->children.push_back(column_declaration->default_expression); columns_list->children.emplace_back(column_declaration); } @@ -1156,6 +1161,10 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) visitor.visit(*create.columns_list); } + // substitute possible UDFs with their definitions + if (!UserDefinedSQLFunctionFactory::instance().empty()) + UserDefinedSQLFunctionVisitor::visit(query_ptr); + /// Set and retrieve list of columns, indices and constraints. Set table engine if needed. Rewrite query in canonical way. TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create); @@ -1226,9 +1235,9 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) /// If table has dependencies - add them to the graph QualifiedTableName qualified_name{database_name, create.getTable()}; - TableNamesSet loading_dependencies = getDependenciesSetFromCreateQuery(getContext()->getGlobalContext(), qualified_name, query_ptr); - if (!loading_dependencies.empty()) - DatabaseCatalog::instance().addLoadingDependencies(qualified_name, std::move(loading_dependencies)); + TableNamesSet dependencies = getLoadingDependenciesFromCreateQuery(getContext()->getGlobalContext(), qualified_name, query_ptr); + if (!dependencies.empty()) + DatabaseCatalog::instance().addDependencies(qualified_name, dependencies); return fillTableIfNeeded(create); } diff --git a/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp b/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp new file mode 100644 index 00000000000..cb237287dc3 --- /dev/null +++ b/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp @@ -0,0 +1,32 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +BlockIO InterpreterDropNamedCollectionQuery::execute() +{ + auto current_context = getContext(); + current_context->checkAccess(AccessType::DROP_NAMED_COLLECTION); + + const auto & query = query_ptr->as(); + if (!query.cluster.empty()) + { + DDLQueryOnClusterParams params; + return executeDDLQueryOnCluster(query_ptr, current_context, params); + } + + if (query.if_exists) + NamedCollectionUtils::removeIfExistsFromSQL(query.collection_name, current_context); + else + NamedCollectionUtils::removeFromSQL(query.collection_name, current_context); + + return {}; +} + +} diff --git a/src/Interpreters/InterpreterDropNamedCollectionQuery.h b/src/Interpreters/InterpreterDropNamedCollectionQuery.h new file mode 100644 index 00000000000..9158bb455d5 --- /dev/null +++ b/src/Interpreters/InterpreterDropNamedCollectionQuery.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace DB +{ + +class Context; + +class InterpreterDropNamedCollectionQuery : public IInterpreter, WithMutableContext +{ +public: + InterpreterDropNamedCollectionQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) {} + + BlockIO execute() override; + +private: + ASTPtr query_ptr; +}; + +} diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 28f8e43ee9b..f237814f879 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -120,6 +121,8 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue auto [database, table] = query.if_exists ? DatabaseCatalog::instance().tryGetDatabaseAndTable(table_id, context_) : DatabaseCatalog::instance().getDatabaseAndTable(table_id, context_); + checkStorageSupportsTransactionsIfNeeded(table, context_); + if (database && table) { auto & ast_drop_query = query.as(); @@ -185,8 +188,8 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue if (query.permanently) { /// Server may fail to restart of DETACH PERMANENTLY if table has dependent ones - DatabaseCatalog::instance().tryRemoveLoadingDependencies(table_id, getContext()->getSettingsRef().check_table_dependencies, - is_drop_or_detach_database); + DatabaseCatalog::instance().removeDependencies(table_id, getContext()->getSettingsRef().check_table_dependencies, + is_drop_or_detach_database); /// Drop table from memory, don't touch data, metadata file renamed and will be skipped during server restart database->detachTablePermanently(context_, table_id.table_name); } @@ -207,18 +210,15 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue table->checkTableCanBeDropped(); - TableExclusiveLockHolder table_lock; - /// We don't need this lock for ReplicatedMergeTree - if (!table->supportsReplication()) - { - /// And for simple MergeTree we can stop merges before acquiring the lock - auto merges_blocker = table->getActionLock(ActionLocks::PartsMerge); - table_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); - } + TableExclusiveLockHolder table_excl_lock; + /// We don't need any lock for ReplicatedMergeTree and for simple MergeTree + /// For the rest of tables types exclusive lock is needed + if (!std::dynamic_pointer_cast(table)) + table_excl_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); auto metadata_snapshot = table->getInMemoryMetadataPtr(); /// Drop table data, don't touch metadata - table->truncate(query_ptr, metadata_snapshot, context_, table_lock); + table->truncate(query_ptr, metadata_snapshot, context_, table_excl_lock); } else if (query.kind == ASTDropQuery::Kind::Drop) { @@ -243,8 +243,8 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue if (database->getUUID() == UUIDHelpers::Nil) table_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); - DatabaseCatalog::instance().tryRemoveLoadingDependencies(table_id, getContext()->getSettingsRef().check_table_dependencies, - is_drop_or_detach_database); + DatabaseCatalog::instance().removeDependencies(table_id, getContext()->getSettingsRef().check_table_dependencies, + is_drop_or_detach_database); database->dropTable(context_, table_id.table_name, query.sync); /// We have to drop mmapio cache when dropping table from Ordinary database @@ -464,4 +464,16 @@ void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr } } +bool InterpreterDropQuery::supportsTransactions() const +{ + /// Enable only for truncate table with MergeTreeData engine + + auto & drop = query_ptr->as(); + + return drop.cluster.empty() + && !drop.temporary + && drop.kind == ASTDropQuery::Kind::Truncate + && drop.table; +} + } diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 2b65039954b..edd84471c22 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -28,6 +28,8 @@ public: static void executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool sync); + bool supportsTransactions() const override; + private: AccessRightsElements getRequiredAccessForDDLOnCluster() const; ASTPtr query_ptr; diff --git a/src/Interpreters/InterpreterExternalDDLQuery.cpp b/src/Interpreters/InterpreterExternalDDLQuery.cpp index 61fbc34784f..5c06ab4b818 100644 --- a/src/Interpreters/InterpreterExternalDDLQuery.cpp +++ b/src/Interpreters/InterpreterExternalDDLQuery.cpp @@ -13,6 +13,7 @@ # include # include # include +# include #endif namespace DB @@ -44,7 +45,7 @@ BlockIO InterpreterExternalDDLQuery::execute() if (arguments.size() != 2 || !arguments[0]->as() || !arguments[1]->as()) throw Exception("MySQL External require two identifier arguments.", ErrorCodes::BAD_ARGUMENTS); - if (external_ddl_query.external_ddl->as()) + if (external_ddl_query.external_ddl->as()) return MySQLInterpreter::InterpreterMySQLDropQuery( external_ddl_query.external_ddl, getContext(), getIdentifierName(arguments[0]), getIdentifierName(arguments[1])).execute(); diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 06d5746af59..e62fca2916e 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -21,6 +21,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -47,6 +50,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -230,6 +236,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); @@ -270,6 +280,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); @@ -314,6 +328,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index 666a674b2c8..82c230ef8e2 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -124,10 +124,10 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c } else { - TableNamesSet dependencies; + std::vector dependencies; if (!exchange_tables) - dependencies = database_catalog.tryRemoveLoadingDependencies(StorageID(elem.from_database_name, elem.from_table_name), - getContext()->getSettingsRef().check_table_dependencies); + dependencies = database_catalog.removeDependencies(StorageID(elem.from_database_name, elem.from_table_name), + getContext()->getSettingsRef().check_table_dependencies); database->renameTable( getContext(), @@ -138,7 +138,7 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c rename.dictionary); if (!dependencies.empty()) - DatabaseCatalog::instance().addLoadingDependencies(QualifiedTableName{elem.to_database_name, elem.to_table_name}, std::move(dependencies)); + DatabaseCatalog::instance().addDependencies(StorageID(elem.to_database_name, elem.to_table_name), dependencies); } } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 2e20cfbd964..4689b234936 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -72,25 +72,27 @@ #include #include +#include #include #include -#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include +#include #include #include #include +#include +#include +#include +#include +#include #include +#include namespace DB @@ -1071,6 +1073,9 @@ static InterpolateDescriptionPtr getInterpolateDescription( static SortDescription getSortDescriptionFromGroupBy(const ASTSelectQuery & query) { + if (!query.groupBy()) + return {}; + SortDescription order_descr; order_descr.reserve(query.groupBy()->children.size()); @@ -1743,7 +1748,8 @@ static void executeMergeAggregatedImpl( const Settings & settings, const NamesAndTypesList & aggregation_keys, const AggregateDescriptions & aggregates, - bool should_produce_results_in_order_of_bucket_number) + bool should_produce_results_in_order_of_bucket_number, + SortDescription group_by_sort_description) { auto keys = aggregation_keys.getNames(); if (has_grouping_sets) @@ -1773,7 +1779,11 @@ static void executeMergeAggregatedImpl( settings.distributed_aggregation_memory_efficient && is_remote_storage, settings.max_threads, settings.aggregation_memory_efficient_merge_threads, - should_produce_results_in_order_of_bucket_number); + should_produce_results_in_order_of_bucket_number, + settings.max_block_size, + settings.aggregation_in_order_max_block_bytes, + std::move(group_by_sort_description), + settings.enable_memory_bound_merging_of_aggregation_results); query_plan.addStep(std::move(merging_aggregated)); } @@ -1837,6 +1847,9 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan( // Let's just choose the safe option since we don't know the value of `to_stage` here. const bool should_produce_results_in_order_of_bucket_number = true; + // It is used to determine if we should use memory bound merging strategy. Maybe it makes sense for projections, but so far this case is just left untouched. + SortDescription group_by_sort_description; + executeMergeAggregatedImpl( query_plan, query_info.projection->aggregate_overflow_row, @@ -1846,7 +1859,8 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan( context_->getSettingsRef(), query_info.projection->aggregation_keys, query_info.projection->aggregate_descriptions, - should_produce_results_in_order_of_bucket_number); + should_produce_results_in_order_of_bucket_number, + std::move(group_by_sort_description)); } } } @@ -2449,6 +2463,26 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac else group_by_info = nullptr; + if (!group_by_info && settings.force_aggregation_in_order) + { + /// Not the most optimal implementation here, but this branch handles very marginal case. + + group_by_sort_description = getSortDescriptionFromGroupBy(getSelectQuery()); + + auto sorting_step = std::make_unique( + query_plan.getCurrentDataStream(), + group_by_sort_description, + 0 /* LIMIT */, + SortingStep::Settings(*context), + settings.optimize_sorting_by_input_stream_properties); + sorting_step->setStepDescription("Enforced sorting for aggregation in order"); + + query_plan.addStep(std::move(sorting_step)); + + group_by_info = std::make_shared( + group_by_sort_description, group_by_sort_description.size(), 1 /* direction */, 0 /* limit */); + } + auto merge_threads = max_streams; auto temporary_data_merge_threads = settings.aggregation_memory_efficient_merge_threads ? static_cast(settings.aggregation_memory_efficient_merge_threads) @@ -2456,8 +2490,8 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); - const bool should_produce_results_in_order_of_bucket_number - = options.to_stage == QueryProcessingStage::WithMergeableState && settings.distributed_aggregation_memory_efficient; + const bool should_produce_results_in_order_of_bucket_number = options.to_stage == QueryProcessingStage::WithMergeableState + && (settings.distributed_aggregation_memory_efficient || settings.enable_memory_bound_merging_of_aggregation_results); auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), @@ -2472,7 +2506,8 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac settings.group_by_use_nulls, std::move(group_by_info), std::move(group_by_sort_description), - should_produce_results_in_order_of_bucket_number); + should_produce_results_in_order_of_bucket_number, + settings.enable_memory_bound_merging_of_aggregation_results); query_plan.addStep(std::move(aggregating_step)); } @@ -2485,8 +2520,14 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool if (query_info.projection && query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) return; + const Settings & settings = context->getSettingsRef(); + + /// Used to determine if we should use memory bound merging strategy. + auto group_by_sort_description + = !query_analyzer->useGroupingSetKey() ? getSortDescriptionFromGroupBy(getSelectQuery()) : SortDescription{}; + const bool should_produce_results_in_order_of_bucket_number = options.to_stage == QueryProcessingStage::WithMergeableState - && context->getSettingsRef().distributed_aggregation_memory_efficient; + && (settings.distributed_aggregation_memory_efficient || settings.enable_memory_bound_merging_of_aggregation_results); executeMergeAggregatedImpl( query_plan, @@ -2497,7 +2538,8 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool context->getSettingsRef(), query_analyzer->aggregationKeys(), query_analyzer->aggregates(), - should_produce_results_in_order_of_bucket_number); + should_produce_results_in_order_of_bucket_number, + std::move(group_by_sort_description)); } @@ -2728,13 +2770,18 @@ void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before { const Settings & settings = context->getSettingsRef(); - auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, context); UInt64 limit_for_distinct = 0; - /// If after this stage of DISTINCT ORDER BY is not executed, + /// If after this stage of DISTINCT, + /// (1) ORDER BY is not executed + /// (2) there is no LIMIT BY (todo: we can check if DISTINCT and LIMIT BY expressions are match) /// then you can get no more than limit_length + limit_offset of different rows. - if ((!query.orderBy() || !before_order) && limit_length <= std::numeric_limits::max() - limit_offset) - limit_for_distinct = limit_length + limit_offset; + if ((!query.orderBy() || !before_order) && !query.limitBy()) + { + auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, context); + if (limit_length <= std::numeric_limits::max() - limit_offset) + limit_for_distinct = limit_length + limit_offset; + } SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); diff --git a/src/Interpreters/JoinSwitcher.cpp b/src/Interpreters/JoinSwitcher.cpp index 5d5a9b27825..996fd1e4ac7 100644 --- a/src/Interpreters/JoinSwitcher.cpp +++ b/src/Interpreters/JoinSwitcher.cpp @@ -7,16 +7,6 @@ namespace DB { -static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, bool nullable) -{ - if (nullable) - JoinCommon::convertColumnToNullable(column); - else - JoinCommon::removeColumnNullability(column); - - return std::move(column); -} - JoinSwitcher::JoinSwitcher(std::shared_ptr table_join_, const Block & right_sample_block_) : limits(table_join_->sizeLimits()) , switched(false) @@ -43,45 +33,25 @@ bool JoinSwitcher::addJoinedBlock(const Block & block, bool) size_t bytes = join->getTotalByteCount(); if (!limits.softCheck(rows, bytes)) - switchJoin(); + return switchJoin(); return true; } -void JoinSwitcher::switchJoin() +bool JoinSwitcher::switchJoin() { - std::shared_ptr joined_data = static_cast(*join).getJoinedData(); - BlocksList right_blocks = std::move(joined_data->blocks); + HashJoin * hash_join = assert_cast(join.get()); + BlocksList right_blocks = hash_join->releaseJoinedBlocks(); - /// Destroy old join & create new one. Early destroy for memory saving. + /// Destroy old join & create new one. join = std::make_shared(table_join, right_sample_block); - /// names to positions optimization - std::vector positions; - std::vector is_nullable; - if (!right_blocks.empty()) - { - positions.reserve(right_sample_block.columns()); - const Block & tmp_block = *right_blocks.begin(); - for (const auto & sample_column : right_sample_block) - { - positions.emplace_back(tmp_block.getPositionByName(sample_column.name)); - is_nullable.emplace_back(JoinCommon::isNullable(sample_column.type)); - } - } - - for (Block & saved_block : right_blocks) - { - Block restored_block; - for (size_t i = 0; i < positions.size(); ++i) - { - auto & column = saved_block.getByPosition(positions[i]); - restored_block.insert(correctNullability(std::move(column), is_nullable[i])); - } - join->addJoinedBlock(restored_block); - } + bool success = true; + for (const Block & saved_block : right_blocks) + success = success && join->addJoinedBlock(saved_block); switched = true; + return success; } } diff --git a/src/Interpreters/JoinSwitcher.h b/src/Interpreters/JoinSwitcher.h index 30115710e22..eec4787037d 100644 --- a/src/Interpreters/JoinSwitcher.h +++ b/src/Interpreters/JoinSwitcher.h @@ -60,12 +60,22 @@ public: return join->alwaysReturnsEmptySet(); } - std::shared_ptr + IBlocksStreamPtr getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const override { return join->getNonJoinedBlocks(left_sample_block, result_sample_block, max_block_size); } + IBlocksStreamPtr getDelayedBlocks() override + { + return join->getDelayedBlocks(); + } + + bool hasDelayedBlocks() const override + { + return join->hasDelayedBlocks(); + } + private: JoinPtr join; SizeLimits limits; @@ -76,7 +86,7 @@ private: /// Change join-in-memory to join-on-disk moving right hand JOIN data from one to another. /// Throws an error if join-on-disk do not support JOIN kind or strictness. - void switchJoin(); + bool switchJoin(); }; } diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 5879c96f7b3..10b122364f9 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -560,11 +560,11 @@ std::vector normalizeColumnNamesExtractNeeded( original_long_name = ident->name(); size_t count = countTablesWithColumn(tables, short_name); + const auto & table = tables[*table_pos]; /// isValidIdentifierBegin retuired to be consistent with TableJoin::deduplicateAndQualifyColumnNames if (count > 1 || aliases.contains(short_name) || !isValidIdentifierBegin(short_name.at(0))) { - const auto & table = tables[*table_pos]; IdentifierSemantic::setColumnLongName(*ident, table.table); /// table.column -> table_alias.column const auto & unique_long_name = ident->name(); @@ -578,6 +578,13 @@ std::vector normalizeColumnNamesExtractNeeded( } else { + if (!table.hasColumn(short_name)) + { + throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, + "There's no column '{}' in table '{}'", + ident->name(), + table.table.getQualifiedNamePrefix(false)); + } ident->setShortName(short_name); /// table.column -> column needed_columns[*table_pos].no_clashes.emplace(short_name); } diff --git a/src/Interpreters/JoinUtils.cpp b/src/Interpreters/JoinUtils.cpp index 59e2475a9b2..d17d3c0d44e 100644 --- a/src/Interpreters/JoinUtils.cpp +++ b/src/Interpreters/JoinUtils.cpp @@ -14,6 +14,11 @@ #include +#include +#include + +#include + namespace DB { @@ -573,6 +578,111 @@ void splitAdditionalColumns(const Names & key_names, const Block & sample_block, } } +template Sharder> +static IColumn::Selector hashToSelector(const WeakHash32 & hash, Sharder sharder) +{ + const auto & hashes = hash.getData(); + size_t num_rows = hashes.size(); + + IColumn::Selector selector(num_rows); + for (size_t i = 0; i < num_rows; ++i) + selector[i] = sharder(intHashCRC32(hashes[i])); + return selector; +} + +template Sharder> +static Blocks scatterBlockByHashImpl(const Strings & key_columns_names, const Block & block, size_t num_shards, Sharder sharder) +{ + size_t num_rows = block.rows(); + size_t num_cols = block.columns(); + + /// Use non-standard initial value so as not to degrade hash map performance inside shard that uses the same CRC32 algorithm. + WeakHash32 hash(num_rows); + for (const auto & key_name : key_columns_names) + { + ColumnPtr key_col = materializeColumn(block, key_name); + key_col->updateWeakHash32(hash); + } + auto selector = hashToSelector(hash, sharder); + + Blocks result; + result.reserve(num_shards); + for (size_t i = 0; i < num_shards; ++i) + { + result.emplace_back(block.cloneEmpty()); + } + + for (size_t i = 0; i < num_cols; ++i) + { + auto dispatched_columns = block.getByPosition(i).column->scatter(num_shards, selector); + assert(result.size() == dispatched_columns.size()); + for (size_t block_index = 0; block_index < num_shards; ++block_index) + { + result[block_index].getByPosition(i).column = std::move(dispatched_columns[block_index]); + } + } + return result; +} + +static Blocks scatterBlockByHashPow2(const Strings & key_columns_names, const Block & block, size_t num_shards) +{ + size_t mask = num_shards - 1; + return scatterBlockByHashImpl(key_columns_names, block, num_shards, [mask](size_t hash) { return hash & mask; }); +} + +static Blocks scatterBlockByHashGeneric(const Strings & key_columns_names, const Block & block, size_t num_shards) +{ + return scatterBlockByHashImpl(key_columns_names, block, num_shards, [num_shards](size_t hash) { return hash % num_shards; }); +} + +Blocks scatterBlockByHash(const Strings & key_columns_names, const Block & block, size_t num_shards) +{ + if (num_shards == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Number of shards must be positive"); + UNUSED(scatterBlockByHashPow2); + // if (likely(isPowerOf2(num_shards))) + // return scatterBlockByHashPow2(key_columns_names, block, num_shards); + return scatterBlockByHashGeneric(key_columns_names, block, num_shards); +} + +template +static Blocks scatterBlockByHashForList(const Strings & key_columns_names, const T & blocks, size_t num_shards) +{ + std::vector scattered_blocks(num_shards); + for (const auto & block : blocks) + { + if (block.rows() == 0) + continue; + auto scattered = scatterBlockByHash(key_columns_names, block, num_shards); + for (size_t i = 0; i < num_shards; ++i) + scattered_blocks[i].emplace_back(std::move(scattered[i])); + } + + Blocks result; + result.reserve(num_shards); + for (size_t i = 0; i < num_shards; ++i) + { + result.emplace_back(concatenateBlocks(scattered_blocks[i])); + } + return result; +} + +Blocks scatterBlockByHash(const Strings & key_columns_names, const Blocks & blocks, size_t num_shards) +{ + return scatterBlockByHashForList(key_columns_names, blocks, num_shards); +} + +Blocks scatterBlockByHash(const Strings & key_columns_names, const BlocksList & blocks, size_t num_shards) +{ + return scatterBlockByHashForList(key_columns_names, blocks, num_shards); +} + +bool hasNonJoinedBlocks(const TableJoin & table_join) +{ + return table_join.strictness() != JoinStrictness::Asof && table_join.strictness() != JoinStrictness::Semi + && isRightOrFull(table_join.kind()); +} + ColumnPtr filterWithBlanks(ColumnPtr src_column, const IColumn::Filter & filter, bool inverse_filter) { ColumnPtr column = src_column->convertToFullColumnIfConst(); @@ -735,7 +845,7 @@ void NotJoinedBlocks::copySameKeys(Block & block) const } } -Block NotJoinedBlocks::read() +Block NotJoinedBlocks::nextImpl() { Block result_block = result_sample_block.cloneEmpty(); { diff --git a/src/Interpreters/JoinUtils.h b/src/Interpreters/JoinUtils.h index 2e26ab782a1..bcff6e60a9a 100644 --- a/src/Interpreters/JoinUtils.h +++ b/src/Interpreters/JoinUtils.h @@ -106,13 +106,19 @@ void splitAdditionalColumns(const Names & key_names, const Block & sample_block, void changeLowCardinalityInplace(ColumnWithTypeAndName & column); +Blocks scatterBlockByHash(const Strings & key_columns_names, const Block & block, size_t num_shards); +Blocks scatterBlockByHash(const Strings & key_columns_names, const Blocks & blocks, size_t num_shards); +Blocks scatterBlockByHash(const Strings & key_columns_names, const BlocksList & blocks, size_t num_shards); + +bool hasNonJoinedBlocks(const TableJoin & table_join); + /// Insert default values for rows marked in filter ColumnPtr filterWithBlanks(ColumnPtr src_column, const IColumn::Filter & filter, bool inverse_filter = false); } /// Creates result from right table data in RIGHT and FULL JOIN when keys are not present in left table. -class NotJoinedBlocks final +class NotJoinedBlocks final : public IBlocksStream { public: using LeftToRightKeyRemap = std::unordered_map; @@ -134,7 +140,7 @@ public: size_t left_columns_count, const LeftToRightKeyRemap & left_to_right_key_remap); - Block read(); + Block nextImpl() override; private: void extractColumnChanges(size_t right_pos, size_t result_pos); diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index bb9c7bf3f90..191372cd545 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -1114,7 +1114,7 @@ private: }; -std::shared_ptr MergeJoin::getNonJoinedBlocks( +IBlocksStreamPtr MergeJoin::getNonJoinedBlocks( const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const { if (table_join->strictness() == JoinStrictness::All && (is_right || is_full)) @@ -1122,7 +1122,7 @@ std::shared_ptr MergeJoin::getNonJoinedBlocks( size_t left_columns_count = left_sample_block.columns(); assert(left_columns_count == result_sample_block.columns() - right_columns_to_add.columns()); auto non_joined = std::make_unique(*this, max_block_size); - return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); + return std::make_unique(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); } return nullptr; } diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index 3ea15d14240..770ca0409bf 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -35,7 +35,7 @@ public: /// Has to be called only after setTotals()/mergeRightBlocks() bool alwaysReturnsEmptySet() const override { return (is_right || is_inner) && min_max_right_blocks.empty(); } - std::shared_ptr getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const override; + IBlocksStreamPtr getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const override; static bool isSupported(const std::shared_ptr & table_join); diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index 4ed22b34e26..6989940323c 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -543,15 +544,29 @@ void InterpreterDropImpl::validate(const InterpreterDropImpl::TQuery & /*query*/ ASTs InterpreterDropImpl::getRewrittenQueries( const InterpreterDropImpl::TQuery & drop_query, ContextPtr context, const String & mapped_to_database, const String & mysql_database) { - const auto & database_name = resolveDatabase(drop_query.getDatabase(), mysql_database, mapped_to_database, context); - - /// Skip drop database|view|dictionary - if (database_name != mapped_to_database || !drop_query.table || drop_query.is_view || drop_query.is_dictionary) + /// Skip drop database|view|dictionary|others + if (drop_query.kind != TQuery::Kind::Table) return {}; - - ASTPtr rewritten_query = drop_query.clone(); - rewritten_query->as()->setDatabase(mapped_to_database); - return ASTs{rewritten_query}; + TQuery::QualifiedNames tables = drop_query.names; + ASTs rewritten_querys; + for (const auto & table: tables) + { + const auto & database_name = resolveDatabase(table.schema, mysql_database, mapped_to_database, context); + if (database_name != mapped_to_database) + continue; + auto rewritten_query = std::make_shared(); + rewritten_query->setTable(table.shortName); + rewritten_query->setDatabase(mapped_to_database); + if (drop_query.is_truncate) + rewritten_query->kind = ASTDropQuery::Kind::Truncate; + else + rewritten_query->kind = ASTDropQuery::Kind::Drop; + rewritten_query->is_view = false; + //To avoid failure, we always set exists + rewritten_query->if_exists = true; + rewritten_querys.push_back(rewritten_query); + } + return rewritten_querys; } void InterpreterRenameImpl::validate(const InterpreterRenameImpl::TQuery & rename_query, ContextPtr /*context*/) diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h index 1ffaacc7dcc..824024e020d 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h @@ -2,11 +2,11 @@ #include #include -#include #include #include #include #include +#include #include #include @@ -17,7 +17,7 @@ namespace MySQLInterpreter { struct InterpreterDropImpl { - using TQuery = ASTDropQuery; + using TQuery = MySQLParser::ASTDropQuery; static void validate(const TQuery & query, ContextPtr context); diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 84f5570349b..cc22ca6597e 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -230,6 +230,7 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as /// Set up memory profiling thread_group->memory_tracker.setProfilerStep(settings.memory_profiler_step); thread_group->memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability); + thread_group->performance_counters.setTraceProfileEvents(settings.trace_profile_events); } thread_group->memory_tracker.setDescription("(for query)"); diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp new file mode 100644 index 00000000000..dc4a2a8e435 --- /dev/null +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -0,0 +1,395 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace +{ + +template +void calculateMax(Max & max, T x) +{ + if (Max(x) > max) + max = x; +} + +template +void calculateMaxAndSum(Max & max, Sum & sum, T x) +{ + sum += x; + if (Max(x) > max) + max = x; +} + +} + +ServerAsynchronousMetrics::ServerAsynchronousMetrics( + ContextPtr global_context_, + int update_period_seconds, + int heavy_metrics_update_period_seconds, + const ProtocolServerMetricsFunc & protocol_server_metrics_func_) + : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_) + , WithContext(global_context_) + , heavy_metric_update_period(heavy_metrics_update_period_seconds) +{} + +void ServerAsynchronousMetrics::updateImpl(AsynchronousMetricValues & new_values, TimePoint update_time, TimePoint current_time) +{ + if (auto mark_cache = getContext()->getMarkCache()) + { + new_values["MarkCacheBytes"] = { mark_cache->weight(), "Total size of mark cache in bytes" }; + new_values["MarkCacheFiles"] = { mark_cache->count(), "Total number of mark files cached in the mark cache" }; + } + + if (auto uncompressed_cache = getContext()->getUncompressedCache()) + { + new_values["UncompressedCacheBytes"] = { uncompressed_cache->weight(), + "Total size of uncompressed cache in bytes. Uncompressed cache does not usually improve the performance and should be mostly avoided." }; + new_values["UncompressedCacheCells"] = { uncompressed_cache->count(), + "Total number of entries in the uncompressed cache. Each entry represents a decompressed block of data. Uncompressed cache does not usually improve performance and should be mostly avoided." }; + } + + if (auto index_mark_cache = getContext()->getIndexMarkCache()) + { + new_values["IndexMarkCacheBytes"] = { index_mark_cache->weight(), "Total size of mark cache for secondary indices in bytes." }; + new_values["IndexMarkCacheFiles"] = { index_mark_cache->count(), "Total number of mark files cached in the mark cache for secondary indices." }; + } + + if (auto index_uncompressed_cache = getContext()->getIndexUncompressedCache()) + { + new_values["IndexUncompressedCacheBytes"] = { index_uncompressed_cache->weight(), + "Total size of uncompressed cache in bytes for secondary indices. Uncompressed cache does not usually improve the performance and should be mostly avoided." }; + new_values["IndexUncompressedCacheCells"] = { index_uncompressed_cache->count(), + "Total number of entries in the uncompressed cache for secondary indices. Each entry represents a decompressed block of data. Uncompressed cache does not usually improve performance and should be mostly avoided." }; + } + + if (auto mmap_cache = getContext()->getMMappedFileCache()) + { + new_values["MMapCacheCells"] = { mmap_cache->count(), + "The number of files opened with `mmap` (mapped in memory)." + " This is used for queries with the setting `local_filesystem_read_method` set to `mmap`." + " The files opened with `mmap` are kept in the cache to avoid costly TLB flushes."}; + } + + { + auto caches = FileCacheFactory::instance().getAll(); + size_t total_bytes = 0; + size_t total_files = 0; + + for (const auto & [_, cache_data] : caches) + { + total_bytes += cache_data->cache->getUsedCacheSize(); + total_files += cache_data->cache->getFileSegmentsNum(); + } + + new_values["FilesystemCacheBytes"] = { total_bytes, + "Total bytes in the `cache` virtual filesystem. This cache is hold on disk." }; + new_values["FilesystemCacheFiles"] = { total_files, + "Total number of cached file segments in the `cache` virtual filesystem. This cache is hold on disk." }; + } + +#if USE_ROCKSDB + if (auto metadata_cache = getContext()->tryGetMergeTreeMetadataCache()) + { + new_values["MergeTreeMetadataCacheSize"] = { metadata_cache->getEstimateNumKeys(), + "The size of the metadata cache for tables. This cache is experimental and not used in production." }; + } +#endif + +#if USE_EMBEDDED_COMPILER + if (auto * compiled_expression_cache = CompiledExpressionCacheFactory::instance().tryGetCache()) + { + new_values["CompiledExpressionCacheBytes"] = { compiled_expression_cache->weight(), + "Total bytes used for the cache of JIT-compiled code." }; + new_values["CompiledExpressionCacheCount"] = { compiled_expression_cache->count(), + "Total entries in the cache of JIT-compiled code." }; + } +#endif + + new_values["Uptime"] = { getContext()->getUptimeSeconds(), + "The server uptime in seconds. It includes the time spent for server initialization before accepting connections." }; + + if (const auto stats = getHashTablesCacheStatistics()) + { + new_values["HashTableStatsCacheEntries"] = { stats->entries, + "The number of entries in the cache of hash table sizes." + " The cache for hash table sizes is used for predictive optimization of GROUP BY." }; + new_values["HashTableStatsCacheHits"] = { stats->hits, + "The number of times the prediction of a hash table size was correct." }; + new_values["HashTableStatsCacheMisses"] = { stats->misses, + "The number of times the prediction of a hash table size was incorrect." }; + } + + /// Free space in filesystems at data path and logs path. + { + auto stat = getStatVFS(getContext()->getPath()); + + new_values["FilesystemMainPathTotalBytes"] = { stat.f_blocks * stat.f_frsize, + "The size of the volume where the main ClickHouse path is mounted, in bytes." }; + new_values["FilesystemMainPathAvailableBytes"] = { stat.f_bavail * stat.f_frsize, + "Available bytes on the volume where the main ClickHouse path is mounted." }; + new_values["FilesystemMainPathUsedBytes"] = { (stat.f_blocks - stat.f_bavail) * stat.f_frsize, + "Used bytes on the volume where the main ClickHouse path is mounted." }; + new_values["FilesystemMainPathTotalINodes"] = { stat.f_files, + "The total number of inodes on the volume where the main ClickHouse path is mounted. If it is less than 25 million, it indicates a misconfiguration." }; + new_values["FilesystemMainPathAvailableINodes"] = { stat.f_favail, + "The number of available inodes on the volume where the main ClickHouse path is mounted. If it is close to zero, it indicates a misconfiguration, and you will get 'no space left on device' even when the disk is not full." }; + new_values["FilesystemMainPathUsedINodes"] = { stat.f_files - stat.f_favail, + "The number of used inodes on the volume where the main ClickHouse path is mounted. This value mostly corresponds to the number of files." }; + } + + { + /// Current working directory of the server is the directory with logs. + auto stat = getStatVFS("."); + + new_values["FilesystemLogsPathTotalBytes"] = { stat.f_blocks * stat.f_frsize, + "The size of the volume where ClickHouse logs path is mounted, in bytes. It's recommended to have at least 10 GB for logs." }; + new_values["FilesystemLogsPathAvailableBytes"] = { stat.f_bavail * stat.f_frsize, + "Available bytes on the volume where ClickHouse logs path is mounted. If this value approaches zero, you should tune the log rotation in the configuration file." }; + new_values["FilesystemLogsPathUsedBytes"] = { (stat.f_blocks - stat.f_bavail) * stat.f_frsize, + "Used bytes on the volume where ClickHouse logs path is mounted." }; + new_values["FilesystemLogsPathTotalINodes"] = { stat.f_files, + "The total number of inodes on the volume where ClickHouse logs path is mounted." }; + new_values["FilesystemLogsPathAvailableINodes"] = { stat.f_favail, + "The number of available inodes on the volume where ClickHouse logs path is mounted." }; + new_values["FilesystemLogsPathUsedINodes"] = { stat.f_files - stat.f_favail, + "The number of used inodes on the volume where ClickHouse logs path is mounted." }; + } + + /// Free and total space on every configured disk. + { + DisksMap disks_map = getContext()->getDisksMap(); + for (const auto & [name, disk] : disks_map) + { + auto total = disk->getTotalSpace(); + + /// Some disks don't support information about the space. + if (!total) + continue; + + auto available = disk->getAvailableSpace(); + auto unreserved = disk->getUnreservedSpace(); + + new_values[fmt::format("DiskTotal_{}", name)] = { total, + "The total size in bytes of the disk (virtual filesystem). Remote filesystems can show a large value like 16 EiB." }; + new_values[fmt::format("DiskUsed_{}", name)] = { total - available, + "Used bytes on the disk (virtual filesystem). Remote filesystems not always provide this information." }; + new_values[fmt::format("DiskAvailable_{}", name)] = { available, + "Available bytes on the disk (virtual filesystem). Remote filesystems can show a large value like 16 EiB." }; + new_values[fmt::format("DiskUnreserved_{}", name)] = { unreserved, + "Available bytes on the disk (virtual filesystem) without the reservations for merges, fetches, and moves. Remote filesystems can show a large value like 16 EiB." }; + } + } + + { + auto databases = DatabaseCatalog::instance().getDatabases(); + + size_t max_queue_size = 0; + size_t max_inserts_in_queue = 0; + size_t max_merges_in_queue = 0; + + size_t sum_queue_size = 0; + size_t sum_inserts_in_queue = 0; + size_t sum_merges_in_queue = 0; + + size_t max_absolute_delay = 0; + size_t max_relative_delay = 0; + + size_t max_part_count_for_partition = 0; + + size_t number_of_databases = databases.size(); + size_t total_number_of_tables = 0; + + size_t total_number_of_bytes = 0; + size_t total_number_of_rows = 0; + size_t total_number_of_parts = 0; + + for (const auto & db : databases) + { + /// Check if database can contain MergeTree tables + if (!db.second->canContainMergeTreeTables()) + continue; + + for (auto iterator = db.second->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) + { + ++total_number_of_tables; + const auto & table = iterator->table(); + if (!table) + continue; + + if (MergeTreeData * table_merge_tree = dynamic_cast(table.get())) + { + const auto & settings = getContext()->getSettingsRef(); + + calculateMax(max_part_count_for_partition, table_merge_tree->getMaxPartsCountAndSizeForPartition().first); + total_number_of_bytes += table_merge_tree->totalBytes(settings).value(); + total_number_of_rows += table_merge_tree->totalRows(settings).value(); + total_number_of_parts += table_merge_tree->getPartsCount(); + } + + if (StorageReplicatedMergeTree * table_replicated_merge_tree = typeid_cast(table.get())) + { + StorageReplicatedMergeTree::Status status; + table_replicated_merge_tree->getStatus(status, false); + + calculateMaxAndSum(max_queue_size, sum_queue_size, status.queue.queue_size); + calculateMaxAndSum(max_inserts_in_queue, sum_inserts_in_queue, status.queue.inserts_in_queue); + calculateMaxAndSum(max_merges_in_queue, sum_merges_in_queue, status.queue.merges_in_queue); + + if (!status.is_readonly) + { + try + { + time_t absolute_delay = 0; + time_t relative_delay = 0; + table_replicated_merge_tree->getReplicaDelays(absolute_delay, relative_delay); + + calculateMax(max_absolute_delay, absolute_delay); + calculateMax(max_relative_delay, relative_delay); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__, + "Cannot get replica delay for table: " + backQuoteIfNeed(db.first) + "." + backQuoteIfNeed(iterator->name())); + } + } + } + } + } + + new_values["ReplicasMaxQueueSize"] = { max_queue_size, "Maximum queue size (in the number of operations like get, merge) across Replicated tables." }; + new_values["ReplicasMaxInsertsInQueue"] = { max_inserts_in_queue, "Maximum number of INSERT operations in the queue (still to be replicated) across Replicated tables." }; + new_values["ReplicasMaxMergesInQueue"] = { max_merges_in_queue, "Maximum number of merge operations in the queue (still to be applied) across Replicated tables." }; + + new_values["ReplicasSumQueueSize"] = { sum_queue_size, "Sum queue size (in the number of operations like get, merge) across Replicated tables." }; + new_values["ReplicasSumInsertsInQueue"] = { sum_inserts_in_queue, "Sum of INSERT operations in the queue (still to be replicated) across Replicated tables." }; + new_values["ReplicasSumMergesInQueue"] = { sum_merges_in_queue, "Sum of merge operations in the queue (still to be applied) across Replicated tables." }; + + new_values["ReplicasMaxAbsoluteDelay"] = { max_absolute_delay, "Maximum difference in seconds between the most fresh replicated part and the most fresh data part still to be replicated, across Replicated tables. A very high value indicates a replica with no data." }; + new_values["ReplicasMaxRelativeDelay"] = { max_relative_delay, "Maximum difference between the replica delay and the delay of the most up-to-date replica of the same table, across Replicated tables." }; + + new_values["MaxPartCountForPartition"] = { max_part_count_for_partition, "Maximum number of parts per partition across all partitions of all tables of MergeTree family. Values larger than 300 indicates misconfiguration, overload, or massive data loading." }; + + new_values["NumberOfDatabases"] = { number_of_databases, "Total number of databases on the server." }; + new_values["NumberOfTables"] = { total_number_of_tables, "Total number of tables summed across the databases on the server, excluding the databases that cannot contain MergeTree tables." + " The excluded database engines are those who generate the set of tables on the fly, like `Lazy`, `MySQL`, `PostgreSQL`, `SQlite`."}; + + new_values["TotalBytesOfMergeTreeTables"] = { total_number_of_bytes, "Total amount of bytes (compressed, including data and indices) stored in all tables of MergeTree family." }; + new_values["TotalRowsOfMergeTreeTables"] = { total_number_of_rows, "Total amount of rows (records) stored in all tables of MergeTree family." }; + new_values["TotalPartsOfMergeTreeTables"] = { total_number_of_parts, "Total amount of data parts in all tables of MergeTree family." + " Numbers larger than 10 000 will negatively affect the server startup time and it may indicate unreasonable choice of the partition key." }; + } + +#if USE_NURAFT + { + auto keeper_dispatcher = getContext()->tryGetKeeperDispatcher(); + if (keeper_dispatcher) + updateKeeperInformation(*keeper_dispatcher, new_values); + } +#endif + + updateHeavyMetricsIfNeeded(current_time, update_time, new_values); +} + +void ServerAsynchronousMetrics::logImpl(AsynchronousMetricValues & new_values) +{ + /// Log the new metrics. + if (auto asynchronous_metric_log = getContext()->getAsynchronousMetricLog()) + asynchronous_metric_log->addValues(new_values); +} + +void ServerAsynchronousMetrics::updateDetachedPartsStats() +{ + DetachedPartsStats current_values{}; + + for (const auto & db : DatabaseCatalog::instance().getDatabases()) + { + if (!db.second->canContainMergeTreeTables()) + continue; + + for (auto iterator = db.second->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) + { + const auto & table = iterator->table(); + if (!table) + continue; + + if (MergeTreeData * table_merge_tree = dynamic_cast(table.get())) + { + for (const auto & detached_part: table_merge_tree->getDetachedParts()) + { + if (!detached_part.valid_name) + continue; + + if (detached_part.prefix.empty()) + ++current_values.detached_by_user; + + ++current_values.count; + } + } + } + } + + detached_parts_stats = current_values; +} + +void ServerAsynchronousMetrics::updateHeavyMetricsIfNeeded(TimePoint current_time, TimePoint update_time, AsynchronousMetricValues & new_values) +{ + const auto time_after_previous_update = current_time - heavy_metric_previous_update_time; + const bool update_heavy_metric = time_after_previous_update >= heavy_metric_update_period || first_run; + + if (update_heavy_metric) + { + heavy_metric_previous_update_time = update_time; + + Stopwatch watch; + + /// Test shows that listing 100000 entries consuming around 0.15 sec. + updateDetachedPartsStats(); + + watch.stop(); + + /// Normally heavy metrics don't delay the rest of the metrics calculation + /// otherwise log the warning message + auto log_level = std::make_pair(DB::LogsLevel::trace, Poco::Message::PRIO_TRACE); + if (watch.elapsedSeconds() > (update_period.count() / 2.)) + log_level = std::make_pair(DB::LogsLevel::debug, Poco::Message::PRIO_DEBUG); + else if (watch.elapsedSeconds() > (update_period.count() / 4. * 3)) + log_level = std::make_pair(DB::LogsLevel::warning, Poco::Message::PRIO_WARNING); + LOG_IMPL(log, log_level.first, log_level.second, + "Update heavy metrics. " + "Update period {} sec. " + "Update heavy metrics period {} sec. " + "Heavy metrics calculation elapsed: {} sec.", + update_period.count(), + heavy_metric_update_period.count(), + watch.elapsedSeconds()); + + } + + + new_values["NumberOfDetachedParts"] = { detached_parts_stats.count, "The total number of parts detached from MergeTree tables. A part can be detached by a user with the `ALTER TABLE DETACH` query or by the server itself it the part is broken, unexpected or unneeded. The server does not care about detached parts and they can be removed." }; + new_values["NumberOfDetachedByUserParts"] = { detached_parts_stats.detached_by_user, "The total number of parts detached from MergeTree tables by users with the `ALTER TABLE DETACH` query (as opposed to unexpected, broken or ignored parts). The server does not care about detached parts and they can be removed." }; +} + +} diff --git a/src/Interpreters/ServerAsynchronousMetrics.h b/src/Interpreters/ServerAsynchronousMetrics.h new file mode 100644 index 00000000000..81047e2fdf9 --- /dev/null +++ b/src/Interpreters/ServerAsynchronousMetrics.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class ServerAsynchronousMetrics : public AsynchronousMetrics, WithContext +{ +public: + ServerAsynchronousMetrics( + ContextPtr global_context_, + int update_period_seconds, + int heavy_metrics_update_period_seconds, + const ProtocolServerMetricsFunc & protocol_server_metrics_func_); +private: + void updateImpl(AsynchronousMetricValues & new_values, TimePoint update_time, TimePoint current_time) override; + void logImpl(AsynchronousMetricValues & new_values) override; + + const Duration heavy_metric_update_period; + TimePoint heavy_metric_previous_update_time; + + struct DetachedPartsStats + { + size_t count; + size_t detached_by_user; + }; + + DetachedPartsStats detached_parts_stats{}; + + void updateDetachedPartsStats(); + void updateHeavyMetricsIfNeeded(TimePoint current_time, TimePoint update_time, AsynchronousMetricValues & new_values); +}; + +} diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 7639dec813d..b6f120edc6c 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -117,6 +117,8 @@ public: if (!thread.joinable()) thread = ThreadFromGlobalPool{&NamedSessionsStorage::cleanThread, this}; + LOG_TRACE(log, "Create new session with session_id: {}, user_id: {}", key.second, key.first); + return {session, true}; } else @@ -124,6 +126,8 @@ public: /// Use existing session. const auto & session = it->second; + LOG_TEST(log, "Reuse session from storage with session_id: {}, user_id: {}", key.second, key.first); + if (!session.unique()) throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); return {session, false}; @@ -173,6 +177,10 @@ private: close_times.resize(close_index + 1); close_times[close_index].emplace_back(session.key); } + + LOG_TEST(log, "Schedule closing session with session_id: {}, user_id: {}", + session.key.second, session.key.first); + } void cleanThread() @@ -214,12 +222,17 @@ private: { if (!session->second.unique()) { + LOG_TEST(log, "Delay closing session with session_id: {}, user_id: {}", key.second, key.first); + /// Skip but move it to close on the next cycle. session->second->timeout = std::chrono::steady_clock::duration{0}; scheduleCloseSession(*session->second, lock); } else + { + LOG_TRACE(log, "Close session with session_id: {}, user_id: {}", key.second, key.first); sessions.erase(session); + } } } @@ -231,6 +244,8 @@ private: std::condition_variable cond; ThreadFromGlobalPool thread; bool quit = false; + + Poco::Logger * log = &Poco::Logger::get("NamedSessionsStorage"); }; @@ -257,11 +272,6 @@ Session::Session(const ContextPtr & global_context_, ClientInfo::Interface inter Session::~Session() { - LOG_DEBUG(log, "{} Destroying {}", - toString(auth_id), - (named_session ? "named session '" + named_session->key.second + "'" : "unnamed session") - ); - /// Early release a NamedSessionData. if (named_session) named_session->release(); diff --git a/src/Interpreters/StorageID.cpp b/src/Interpreters/StorageID.cpp index 8811adc087b..70dea02ccc5 100644 --- a/src/Interpreters/StorageID.cpp +++ b/src/Interpreters/StorageID.cpp @@ -64,21 +64,8 @@ String StorageID::getNameForLogs() const + (hasUUID() ? " (" + toString(uuid) + ")" : ""); } -bool StorageID::operator<(const StorageID & rhs) const -{ - assertNotEmpty(); - /// It's needed for ViewDependencies - if (!hasUUID() && !rhs.hasUUID()) - /// If both IDs don't have UUID, compare them like pair of strings - return std::tie(database_name, table_name) < std::tie(rhs.database_name, rhs.table_name); - else if (hasUUID() && rhs.hasUUID()) - /// If both IDs have UUID, compare UUIDs and ignore database and table name - return uuid < rhs.uuid; - else - /// All IDs without UUID are less, then all IDs with UUID - return !hasUUID(); -} - +/// NOTE: This implementation doesn't allow to implement a good "operator <". +/// Because "a != b" must be equivalent to "(a < b) || (b < a)", and we can't make "operator <" to meet that. bool StorageID::operator==(const StorageID & rhs) const { assertNotEmpty(); diff --git a/src/Interpreters/StorageID.h b/src/Interpreters/StorageID.h index 43710988243..68c83f753b5 100644 --- a/src/Interpreters/StorageID.h +++ b/src/Interpreters/StorageID.h @@ -45,6 +45,8 @@ struct StorageID StorageID(const ASTTableIdentifier & table_identifier_node); /// NOLINT StorageID(const ASTPtr & node); /// NOLINT + explicit StorageID(const QualifiedTableName & qualified_name) : StorageID(qualified_name.database, qualified_name.table) { } + String getDatabaseName() const; String getTableName() const; @@ -71,7 +73,6 @@ struct StorageID bool hasDatabase() const { return !database_name.empty(); } - bool operator<(const StorageID & rhs) const; bool operator==(const StorageID & rhs) const; void assertNotEmpty() const @@ -97,8 +98,47 @@ struct StorageID /// Get short, but unique, name. String getShortName() const; + /// Calculates hash using only the database and table name of a StorageID. + struct DatabaseAndTableNameHash + { + size_t operator()(const StorageID & storage_id) const + { + SipHash hash_state; + hash_state.update(storage_id.database_name.data(), storage_id.database_name.size()); + hash_state.update(storage_id.table_name.data(), storage_id.table_name.size()); + return hash_state.get64(); + } + }; + + /// Checks if the database and table name of two StorageIDs are equal. + struct DatabaseAndTableNameEqual + { + bool operator()(const StorageID & left, const StorageID & right) const + { + return (left.database_name == right.database_name) && (left.table_name == right.table_name); + } + }; + private: StorageID() = default; }; } + +namespace fmt +{ + template <> + struct formatter + { + static constexpr auto parse(format_parse_context & ctx) + { + return ctx.begin(); + } + + template + auto format(const DB::StorageID & storage_id, FormatContext & ctx) + { + return format_to(ctx.out(), "{}", storage_id.getNameForLogs()); + } + }; +} diff --git a/src/Interpreters/TraceCollector.cpp b/src/Interpreters/TraceCollector.cpp index 41a7fcf8389..050dea02717 100644 --- a/src/Interpreters/TraceCollector.cpp +++ b/src/Interpreters/TraceCollector.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -96,6 +97,12 @@ void TraceCollector::run() Int64 size; readPODBinary(size, in); + ProfileEvents::Event event; + readPODBinary(event, in); + + ProfileEvents::Count increment; + readPODBinary(increment, in); + if (trace_log) { // time and time_in_microseconds are both being constructed from the same timespec so that the @@ -105,7 +112,7 @@ void TraceCollector::run() UInt64 time = static_cast(ts.tv_sec * 1000000000LL + ts.tv_nsec); UInt64 time_in_microseconds = static_cast((ts.tv_sec * 1000000LL) + (ts.tv_nsec / 1000)); - TraceLogElement element{time_t(time / 1000000000), time_in_microseconds, time, trace_type, thread_id, query_id, trace, size}; + TraceLogElement element{time_t(time / 1000000000), time_in_microseconds, time, trace_type, thread_id, query_id, trace, size, event, increment}; trace_log->add(element); } } diff --git a/src/Interpreters/TraceCollector.h b/src/Interpreters/TraceCollector.h index b3f11ca5756..40fa854b791 100644 --- a/src/Interpreters/TraceCollector.h +++ b/src/Interpreters/TraceCollector.h @@ -1,7 +1,5 @@ #pragma once - #include -#include class StackTrace; @@ -21,11 +19,6 @@ public: explicit TraceCollector(std::shared_ptr trace_log_); ~TraceCollector(); - static inline void collect(TraceType trace_type, const StackTrace & stack_trace, Int64 size) - { - return TraceSender::send(trace_type, stack_trace, size); - } - private: std::shared_ptr trace_log; ThreadFromGlobalPool thread; diff --git a/src/Interpreters/TraceLog.cpp b/src/Interpreters/TraceLog.cpp index c16a73e75dc..0408ebe504b 100644 --- a/src/Interpreters/TraceLog.cpp +++ b/src/Interpreters/TraceLog.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -20,6 +21,7 @@ const TraceDataType::Values TraceLogElement::trace_values = {"Memory", static_cast(TraceType::Memory)}, {"MemorySample", static_cast(TraceType::MemorySample)}, {"MemoryPeak", static_cast(TraceType::MemoryPeak)}, + {"ProfileEvent", static_cast(TraceType::ProfileEvent)}, }; NamesAndTypesList TraceLogElement::getNamesAndTypes() @@ -36,6 +38,8 @@ NamesAndTypesList TraceLogElement::getNamesAndTypes() {"query_id", std::make_shared()}, {"trace", std::make_shared(std::make_shared())}, {"size", std::make_shared()}, + {"event", std::make_shared(std::make_shared())}, + {"increment", std::make_shared()}, }; } @@ -53,6 +57,13 @@ void TraceLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insertData(query_id.data(), query_id.size()); columns[i++]->insert(trace); columns[i++]->insert(size); + + String event_name; + if (event != ProfileEvents::end()) + event_name = ProfileEvents::getName(event); + + columns[i++]->insert(event_name); + columns[i++]->insert(increment); } } diff --git a/src/Interpreters/TraceLog.h b/src/Interpreters/TraceLog.h index 43d7861327f..c481f033a72 100644 --- a/src/Interpreters/TraceLog.h +++ b/src/Interpreters/TraceLog.h @@ -3,8 +3,9 @@ #include #include #include -#include #include +#include +#include #include #include @@ -26,7 +27,12 @@ struct TraceLogElement UInt64 thread_id{}; String query_id{}; Array trace{}; - Int64 size{}; /// Allocation size in bytes for TraceType::Memory + /// Allocation size in bytes for TraceType::Memory. + Int64 size{}; + /// ProfileEvent for TraceType::ProfileEvent. + ProfileEvents::Event event{ProfileEvents::end()}; + /// Increment of profile event for TraceType::ProfileEvent. + ProfileEvents::Count increment{}; static std::string name() { return "TraceLog"; } static NamesAndTypesList getNamesAndTypes(); diff --git a/src/Interpreters/TransactionVersionMetadata.cpp b/src/Interpreters/TransactionVersionMetadata.cpp index 5f46b86508c..7671f512bdc 100644 --- a/src/Interpreters/TransactionVersionMetadata.cpp +++ b/src/Interpreters/TransactionVersionMetadata.cpp @@ -243,6 +243,9 @@ bool VersionMetadata::canBeRemoved() { /// Avoid access to Transaction log if transactions are not involved + if (creation_csn.load(std::memory_order_relaxed) == Tx::RolledBackCSN) + return true; + TIDHash removal_lock = removal_tid_lock.load(std::memory_order_relaxed); if (!removal_lock) return false; @@ -380,8 +383,9 @@ void VersionMetadata::read(ReadBuffer & buf) if (name == CREATION_CSN_STR) { - chassert(!creation_csn); - creation_csn = read_csn(); + auto new_val = read_csn(); + chassert(!creation_csn || (creation_csn == new_val && creation_csn == Tx::PrehistoricCSN)); + creation_csn = new_val; } else if (name == REMOVAL_TID_STR) { diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index e4301bad1e8..182e9623c61 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -246,36 +246,6 @@ GroupByKeysInfo getGroupByKeysInfo(const ASTs & group_by_keys) return data; } -///eliminate functions of other GROUP BY keys -void optimizeGroupByFunctionKeys(ASTSelectQuery * select_query) -{ - if (!select_query->groupBy()) - return; - - auto group_by = select_query->groupBy(); - const auto & group_by_keys = group_by->children; - - ASTs modified; ///result - - GroupByKeysInfo group_by_keys_data = getGroupByKeysInfo(group_by_keys); - - if (!group_by_keys_data.has_function) - return; - - GroupByFunctionKeysVisitor::Data visitor_data{group_by_keys_data.key_names}; - GroupByFunctionKeysVisitor(visitor_data).visit(group_by); - - modified.reserve(group_by_keys.size()); - - /// filling the result - for (const auto & group_key : group_by_keys) - if (group_by_keys_data.key_names.contains(group_key->getColumnName())) - modified.push_back(group_key); - - /// modifying the input - group_by->children = modified; -} - /// Eliminates min/max/any-aggregators of functions of GROUP BY keys void optimizeAggregateFunctionsOfGroupByKeys(ASTSelectQuery * select_query, ASTPtr & node) { @@ -793,6 +763,36 @@ void TreeOptimizer::optimizeCountConstantAndSumOne(ASTPtr & query) RewriteCountVariantsVisitor::visit(query); } +///eliminate functions of other GROUP BY keys +void TreeOptimizer::optimizeGroupByFunctionKeys(ASTSelectQuery * select_query) +{ + if (!select_query->groupBy()) + return; + + auto group_by = select_query->groupBy(); + const auto & group_by_keys = group_by->children; + + ASTs modified; ///result + + GroupByKeysInfo group_by_keys_data = getGroupByKeysInfo(group_by_keys); + + if (!group_by_keys_data.has_function) + return; + + GroupByFunctionKeysVisitor::Data visitor_data{group_by_keys_data.key_names}; + GroupByFunctionKeysVisitor(visitor_data).visit(group_by); + + modified.reserve(group_by_keys.size()); + + /// filling the result + for (const auto & group_key : group_by_keys) + if (group_by_keys_data.key_names.contains(group_key->getColumnName())) + modified.push_back(group_key); + + /// modifying the input + group_by->children = modified; +} + void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, const std::vector & tables_with_columns, ContextPtr context) { diff --git a/src/Interpreters/TreeOptimizer.h b/src/Interpreters/TreeOptimizer.h index ced185373cc..72a240d83b5 100644 --- a/src/Interpreters/TreeOptimizer.h +++ b/src/Interpreters/TreeOptimizer.h @@ -25,6 +25,7 @@ public: static void optimizeIf(ASTPtr & query, Aliases & aliases, bool if_chain_to_multiif); static void optimizeCountConstantAndSumOne(ASTPtr & query); + static void optimizeGroupByFunctionKeys(ASTSelectQuery * select_query); }; } diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 30fab527ac5..828f332af1d 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -1359,7 +1359,9 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( TreeOptimizer::optimizeIf(query, result.aliases, settings.optimize_if_chain_to_multiif); /// Only apply AST optimization for initial queries. - if (getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && !select_options.ignore_ast_optimizations) + const bool ast_optimizations_allowed + = getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && !select_options.ignore_ast_optimizations; + if (ast_optimizations_allowed) TreeOptimizer::apply(query, result, tables_with_columns, getContext()); /// array_join_alias_to_name, array_join_result_to_source. @@ -1396,6 +1398,10 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( /// If query is changed, we need to redo some work to correct name resolution. if (is_changed) { + /// We should re-apply the optimization, because an expression substituted from alias column might be a function of a group key. + if (ast_optimizations_allowed && settings.optimize_group_by_function_keys) + TreeOptimizer::optimizeGroupByFunctionKeys(select_query); + result.aggregates = getAggregates(query, *select_query); result.window_function_asts = getWindowFunctions(query, *select_query); result.expressions_with_window_function = getExpressionsWithWindowFunctions(query); @@ -1473,10 +1479,7 @@ void TreeRewriter::normalize( ASTPtr & query, Aliases & aliases, const NameSet & source_columns_set, bool ignore_alias, const Settings & settings, bool allow_self_aliases, ContextPtr context_) { if (!UserDefinedSQLFunctionFactory::instance().empty()) - { - UserDefinedSQLFunctionVisitor::Data data_user_defined_functions_visitor; - UserDefinedSQLFunctionVisitor(data_user_defined_functions_visitor).visit(query); - } + UserDefinedSQLFunctionVisitor::visit(query); CustomizeCountDistinctVisitor::Data data_count_distinct{settings.count_distinct_implementation}; CustomizeCountDistinctVisitor(data_count_distinct).visit(query); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index b44db316f90..c2680e27444 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -592,13 +592,12 @@ static std::tuple executeQueryImpl( quota->checkExceeded(QuotaType::ERRORS); } - queue->push(ast, context); + auto insert_future = queue->push(ast, context); if (settings.wait_for_async_insert) { auto timeout = settings.wait_for_async_insert_timeout.totalMilliseconds(); - auto query_id = context->getCurrentQueryId(); - auto source = std::make_shared(query_id, timeout, *queue); + auto source = std::make_shared(std::move(insert_future), timeout); res.pipeline = QueryPipeline(Pipe(std::move(source))); } diff --git a/src/Interpreters/loadMetadata.h b/src/Interpreters/loadMetadata.h index b229a2b4c31..3553011fe4d 100644 --- a/src/Interpreters/loadMetadata.h +++ b/src/Interpreters/loadMetadata.h @@ -1,7 +1,6 @@ #pragma once #include -#include namespace DB diff --git a/src/Parsers/ASTAlterNamedCollectionQuery.cpp b/src/Parsers/ASTAlterNamedCollectionQuery.cpp new file mode 100644 index 00000000000..7e95147ad75 --- /dev/null +++ b/src/Parsers/ASTAlterNamedCollectionQuery.cpp @@ -0,0 +1,54 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +ASTPtr ASTAlterNamedCollectionQuery::clone() const +{ + return std::make_shared(*this); +} + +void ASTAlterNamedCollectionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +{ + settings.ostr << (settings.hilite ? hilite_keyword : "") << "Alter NAMED COLLECTION "; + settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(collection_name) << (settings.hilite ? hilite_none : ""); + formatOnCluster(settings); + if (!changes.empty()) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " SET " << (settings.hilite ? hilite_none : ""); + bool first = true; + for (const auto & change : changes) + { + if (!first) + settings.ostr << ", "; + else + first = false; + + formatSettingName(change.name, settings.ostr); + if (settings.show_secrets) + settings.ostr << " = " << applyVisitor(FieldVisitorToString(), change.value); + else + settings.ostr << " = '[HIDDEN]'"; + } + } + if (!delete_keys.empty()) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " DELETE " << (settings.hilite ? hilite_none : ""); + bool first = true; + for (const auto & key : delete_keys) + { + if (!first) + settings.ostr << ", "; + else + first = false; + + formatSettingName(key, settings.ostr); + } + } +} + +} diff --git a/src/Parsers/ASTAlterNamedCollectionQuery.h b/src/Parsers/ASTAlterNamedCollectionQuery.h new file mode 100644 index 00000000000..a8aa06200fd --- /dev/null +++ b/src/Parsers/ASTAlterNamedCollectionQuery.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class ASTAlterNamedCollectionQuery : public IAST, public ASTQueryWithOnCluster +{ +public: + std::string collection_name; + SettingsChanges changes; + std::vector delete_keys; + bool if_exists = false; + + String getID(char) const override { return "AlterNamedCollectionQuery"; } + + ASTPtr clone() const override; + + void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + + ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } +}; + +} diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 959fc55c945..80801278963 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -509,7 +509,7 @@ bool ASTAlterQuery::isOneCommandTypeOnly(const ASTAlterCommand::Type & type) con bool ASTAlterQuery::isSettingsAlter() const { - return isOneCommandTypeOnly(ASTAlterCommand::MODIFY_SETTING); + return isOneCommandTypeOnly(ASTAlterCommand::MODIFY_SETTING) || isOneCommandTypeOnly(ASTAlterCommand::RESET_SETTING); } bool ASTAlterQuery::isFreezeAlter() const diff --git a/src/Parsers/ASTCreateNamedCollectionQuery.cpp b/src/Parsers/ASTCreateNamedCollectionQuery.cpp new file mode 100644 index 00000000000..97e83541f05 --- /dev/null +++ b/src/Parsers/ASTCreateNamedCollectionQuery.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +ASTPtr ASTCreateNamedCollectionQuery::clone() const +{ + return std::make_shared(*this); +} + +void ASTCreateNamedCollectionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +{ + settings.ostr << (settings.hilite ? hilite_keyword : "") << "CREATE NAMED COLLECTION "; + settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(collection_name) << (settings.hilite ? hilite_none : ""); + + formatOnCluster(settings); + + settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); + bool first = true; + for (const auto & change : changes) + { + if (!first) + settings.ostr << ", "; + else + first = false; + + formatSettingName(change.name, settings.ostr); + + if (settings.show_secrets) + settings.ostr << " = " << applyVisitor(FieldVisitorToString(), change.value); + else + settings.ostr << " = '[HIDDEN]'"; + } +} + +} diff --git a/src/Parsers/ASTCreateNamedCollectionQuery.h b/src/Parsers/ASTCreateNamedCollectionQuery.h new file mode 100644 index 00000000000..901e6b50a4c --- /dev/null +++ b/src/Parsers/ASTCreateNamedCollectionQuery.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class ASTCreateNamedCollectionQuery : public IAST, public ASTQueryWithOnCluster +{ +public: + std::string collection_name; + SettingsChanges changes; + + String getID(char) const override { return "CreateNamedCollectionQuery"; } + + ASTPtr clone() const override; + + void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + + ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } + + std::string getCollectionName() const; +}; + +} diff --git a/src/Parsers/ASTDropNamedCollectionQuery.cpp b/src/Parsers/ASTDropNamedCollectionQuery.cpp new file mode 100644 index 00000000000..3b8568cfd70 --- /dev/null +++ b/src/Parsers/ASTDropNamedCollectionQuery.cpp @@ -0,0 +1,20 @@ +#include +#include +#include + +namespace DB +{ + +ASTPtr ASTDropNamedCollectionQuery::clone() const +{ + return std::make_shared(*this); +} + +void ASTDropNamedCollectionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const +{ + settings.ostr << (settings.hilite ? hilite_keyword : "") << "DROP NAMED COLLECTION "; + settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(collection_name) << (settings.hilite ? hilite_none : ""); + formatOnCluster(settings); +} + +} diff --git a/src/Parsers/ASTDropNamedCollectionQuery.h b/src/Parsers/ASTDropNamedCollectionQuery.h new file mode 100644 index 00000000000..0b71bdaf213 --- /dev/null +++ b/src/Parsers/ASTDropNamedCollectionQuery.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class ASTDropNamedCollectionQuery : public IAST, public ASTQueryWithOnCluster +{ +public: + std::string collection_name; + bool if_exists = false; + + String getID(char) const override { return "DropNamedCollectionQuery"; } + + ASTPtr clone() const override; + + void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + + ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } +}; + +} diff --git a/src/Parsers/MySQL/ASTDropQuery.cpp b/src/Parsers/MySQL/ASTDropQuery.cpp new file mode 100644 index 00000000000..fb76d93363a --- /dev/null +++ b/src/Parsers/MySQL/ASTDropQuery.cpp @@ -0,0 +1,119 @@ +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace MySQLParser +{ + +ASTPtr ASTDropQuery::clone() const +{ + auto res = std::make_shared(*this); + res->children.clear(); + res->is_truncate = is_truncate; + res->if_exists = if_exists; + return res; +} + +bool ParserDropQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_drop("DROP"); + ParserKeyword s_truncate("TRUNCATE"); + ParserKeyword s_table("TABLE"); + ParserKeyword s_database("DATABASE"); + ParserKeyword s_if_exists("IF EXISTS"); + ParserKeyword s_view("VIEW"); + ParserKeyword on("ON"); + ParserIdentifier name_p(false); + + ParserKeyword s_event("EVENT"); + ParserKeyword s_function("FUNCTION"); + ParserKeyword s_index("INDEX"); + ParserKeyword s_server("SERVER"); + ParserKeyword s_trigger("TRIGGER"); + + auto query = std::make_shared(); + node = query; + ASTDropQuery::QualifiedNames names; + bool if_exists = false; + bool is_truncate = false; + + if (s_truncate.ignore(pos, expected) && s_table.ignore(pos, expected)) + { + is_truncate = true; + query->kind = ASTDropQuery::Kind::Table; + ASTDropQuery::QualifiedName name; + if (parseDatabaseAndTableName(pos, expected, name.schema, name.shortName)) + names.push_back(name); + else + return false; + } + else if (s_drop.ignore(pos, expected)) + { + if (s_database.ignore(pos, expected)) + { + query->kind = ASTDropQuery::Kind::Database; + if (s_if_exists.ignore(pos, expected)) + if_exists = true; + ASTPtr database; + if (!name_p.parse(pos, database, expected)) + return false; + } + else + { + if (s_view.ignore(pos, expected)) + query->kind = ASTDropQuery::Kind::View; + else if (s_table.ignore(pos, expected)) + query->kind = ASTDropQuery::Kind::Table; + else if (s_index.ignore(pos, expected)) + { + ASTPtr index; + query->kind = ASTDropQuery::Kind::Index; + if (!(name_p.parse(pos, index, expected) && on.ignore(pos, expected))) + return false; + } + else if (s_event.ignore(pos, expected) || s_function.ignore(pos, expected) || s_server.ignore(pos, expected) + || s_trigger.ignore(pos, expected)) + { + query->kind = ASTDropQuery::Kind::Other; + } + else + return false; + + if (s_if_exists.ignore(pos, expected)) + if_exists = true; + //parse name + auto parse_element = [&] + { + ASTDropQuery::QualifiedName element; + if (parseDatabaseAndTableName(pos, expected, element.schema, element.shortName)) + { + names.emplace_back(std::move(element)); + return true; + } + return false; + }; + + if (!ParserList::parseUtil(pos, expected, parse_element, false)) + return false; + } + } + else + return false; + + query->if_exists = if_exists; + query->names = names; + query->is_truncate = is_truncate; + + return true; +} + +} + +} diff --git a/src/Parsers/MySQL/ASTDropQuery.h b/src/Parsers/MySQL/ASTDropQuery.h new file mode 100644 index 00000000000..ff95277ae5e --- /dev/null +++ b/src/Parsers/MySQL/ASTDropQuery.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + +namespace MySQLParser +{ + +class ASTDropQuery : public IAST +{ +public: + enum Kind + { + Table, + View, + Database, + Index, + /// TRIGGER,FUNCTION,EVENT and so on, No need for support + Other, + }; + Kind kind; + struct QualifiedName + { + String schema; + String shortName; + }; + + using QualifiedNames = std::vector; + QualifiedNames names; + bool if_exists{false}; + //drop or truncate + bool is_truncate{false}; + + ASTPtr clone() const override; + String getID(char /*delim*/) const override {return "ASTDropQuery" ;} + +protected: + void formatImpl(const FormatSettings & /*settings*/, FormatState & /*state*/, FormatStateStacked /*frame*/) const override + { + throw Exception("Method formatImpl is not supported by MySQLParser::ASTDropQuery.", ErrorCodes::NOT_IMPLEMENTED); + } +}; + +class ParserDropQuery : public IParserBase +{ +protected: + const char * getName() const override { return "DROP query"; } + + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} + +} diff --git a/src/Parsers/ParserAlterNamedCollectionQuery.cpp b/src/Parsers/ParserAlterNamedCollectionQuery.cpp new file mode 100644 index 00000000000..9108747ad82 --- /dev/null +++ b/src/Parsers/ParserAlterNamedCollectionQuery.cpp @@ -0,0 +1,85 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +bool ParserAlterNamedCollectionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_alter("ALTER"); + ParserKeyword s_collection("NAMED COLLECTION"); + ParserKeyword s_delete("DELETE"); + + ParserIdentifier name_p; + ParserSetQuery set_p; + ParserToken s_comma(TokenType::Comma); + + String cluster_str; + bool if_exists = false; + + ASTPtr collection_name; + ASTPtr set; + std::vector delete_keys; + + if (!s_alter.ignore(pos, expected)) + return false; + + if (!s_collection.ignore(pos, expected)) + return false; + + if (!name_p.parse(pos, collection_name, expected)) + return false; + + if (ParserKeyword{"ON"}.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + } + + bool parsed_delete = false; + if (!set_p.parse(pos, set, expected)) + { + if (!s_delete.ignore(pos, expected)) + return false; + + parsed_delete = true; + } + else if (s_delete.ignore(pos, expected)) + { + parsed_delete = true; + } + + if (parsed_delete) + { + while (true) + { + if (!delete_keys.empty() && !s_comma.ignore(pos)) + break; + + ASTPtr key; + if (!name_p.parse(pos, key, expected)) + return false; + + delete_keys.push_back(getIdentifierName(key)); + } + } + + auto query = std::make_shared(); + + query->collection_name = getIdentifierName(collection_name); + query->if_exists = if_exists; + query->cluster = std::move(cluster_str); + if (set) + query->changes = set->as()->changes; + query->delete_keys = delete_keys; + + node = query; + return true; +} + +} diff --git a/src/Parsers/ParserAlterNamedCollectionQuery.h b/src/Parsers/ParserAlterNamedCollectionQuery.h new file mode 100644 index 00000000000..66ad61447dd --- /dev/null +++ b/src/Parsers/ParserAlterNamedCollectionQuery.h @@ -0,0 +1,14 @@ +#pragma once + +#include "IParserBase.h" + +namespace DB +{ + +class ParserAlterNamedCollectionQuery : public IParserBase +{ +protected: + const char * getName() const override { return "Alter NAMED COLLECTION query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; +} diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 9c1c682ca03..90df8a8f79a 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -1383,6 +1384,59 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec } +bool ParserCreateNamedCollectionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_create("CREATE"); + ParserKeyword s_attach("ATTACH"); + ParserKeyword s_named_collection("NAMED COLLECTION"); + ParserKeyword s_as("AS"); + + ParserToken s_comma(TokenType::Comma); + ParserIdentifier name_p; + + ASTPtr collection_name; + String cluster_str; + + if (!s_create.ignore(pos, expected)) + return false; + + if (!s_named_collection.ignore(pos, expected)) + return false; + + if (!name_p.parse(pos, collection_name, expected)) + return false; + + if (ParserKeyword{"ON"}.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + } + + if (!s_as.ignore(pos, expected)) + return false; + + SettingsChanges changes; + + while (true) + { + if (!changes.empty() && !s_comma.ignore(pos)) + break; + + changes.push_back(SettingChange{}); + + if (!ParserSetQuery::parseNameValuePair(changes.back(), pos, expected)) + return false; + } + + auto query = std::make_shared(); + + tryGetIdentifierNameInto(collection_name, query->collection_name); + query->changes = changes; + + node = query; + return true; +} + bool ParserCreateDictionaryQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_create("CREATE"); diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index e1573c92dab..e97033c51f0 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -522,6 +522,13 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserCreateNamedCollectionQuery : public IParserBase +{ +protected: + const char * getName() const override { return "CREATE NAMED COLLECTION"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + /** Query like this: * CREATE|ATTACH TABLE [IF NOT EXISTS] [db.]name diff --git a/src/Parsers/ParserDropNamedCollectionQuery.cpp b/src/Parsers/ParserDropNamedCollectionQuery.cpp new file mode 100644 index 00000000000..1ea8aa6d75d --- /dev/null +++ b/src/Parsers/ParserDropNamedCollectionQuery.cpp @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +bool ParserDropNamedCollectionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_drop("DROP"); + ParserKeyword s_collection("NAMED COLLECTION"); + ParserKeyword s_if_exists("IF EXISTS"); + ParserIdentifier name_p; + + String cluster_str; + bool if_exists = false; + + ASTPtr collection_name; + + if (!s_drop.ignore(pos, expected)) + return false; + + if (!s_collection.ignore(pos, expected)) + return false; + + if (s_if_exists.ignore(pos, expected)) + if_exists = true; + + if (!name_p.parse(pos, collection_name, expected)) + return false; + + if (ParserKeyword{"ON"}.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + } + + auto query = std::make_shared(); + + tryGetIdentifierNameInto(collection_name, query->collection_name); + query->if_exists = if_exists; + query->cluster = std::move(cluster_str); + + node = query; + return true; +} + +} diff --git a/src/Parsers/ParserDropNamedCollectionQuery.h b/src/Parsers/ParserDropNamedCollectionQuery.h new file mode 100644 index 00000000000..5dd3ef63e05 --- /dev/null +++ b/src/Parsers/ParserDropNamedCollectionQuery.h @@ -0,0 +1,14 @@ +#pragma once + +#include "IParserBase.h" + +namespace DB +{ + +class ParserDropNamedCollectionQuery : public IParserBase +{ +protected: + const char * getName() const override { return "DROP NAMED COLLECTION query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; +} diff --git a/src/Parsers/ParserExternalDDLQuery.cpp b/src/Parsers/ParserExternalDDLQuery.cpp index 839838c4f54..5d6874f524d 100644 --- a/src/Parsers/ParserExternalDDLQuery.cpp +++ b/src/Parsers/ParserExternalDDLQuery.cpp @@ -11,6 +11,7 @@ #if USE_MYSQL # include # include +# include #endif namespace DB @@ -43,7 +44,7 @@ bool ParserExternalDDLQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expect if (external_ddl_query->from->name == "MySQL") { #if USE_MYSQL - ParserDropQuery p_drop_query; + MySQLParser::ParserDropQuery p_drop_query; ParserRenameQuery p_rename_query; MySQLParser::ParserAlterQuery p_alter_query; MySQLParser::ParserCreateQuery p_create_query; diff --git a/src/Parsers/ParserQuery.cpp b/src/Parsers/ParserQuery.cpp index ca837e7dcc5..77e7b58e6b1 100644 --- a/src/Parsers/ParserQuery.cpp +++ b/src/Parsers/ParserQuery.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include @@ -46,6 +48,9 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserCreateSettingsProfileQuery create_settings_profile_p; ParserCreateFunctionQuery create_function_p; ParserDropFunctionQuery drop_function_p; + ParserCreateNamedCollectionQuery create_named_collection_p; + ParserDropNamedCollectionQuery drop_named_collection_p; + ParserAlterNamedCollectionQuery alter_named_collection_p; ParserCreateIndexQuery create_index_p; ParserDropIndexQuery drop_index_p; ParserDropAccessEntityQuery drop_access_entity_p; @@ -69,6 +74,9 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) || create_settings_profile_p.parse(pos, node, expected) || create_function_p.parse(pos, node, expected) || drop_function_p.parse(pos, node, expected) + || create_named_collection_p.parse(pos, node, expected) + || drop_named_collection_p.parse(pos, node, expected) + || alter_named_collection_p.parse(pos, node, expected) || create_index_p.parse(pos, node, expected) || drop_index_p.parse(pos, node, expected) || drop_access_entity_p.parse(pos, node, expected) diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 201cd750af8..107db51f869 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -108,6 +108,13 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } } + /// FROM database.table or FROM table or FROM (subquery) or FROM tableFunction(...) + if (s_from.ignore(pos, expected)) + { + if (!ParserTablesInSelectQuery(false).parse(pos, tables, expected)) + return false; + } + /// SELECT [ALL/DISTINCT [ON (expr_list)]] [TOP N [WITH TIES]] expr_list { bool has_all = false; @@ -166,7 +173,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } /// FROM database.table or FROM table or FROM (subquery) or FROM tableFunction(...) - if (s_from.ignore(pos, expected)) + if (!tables && s_from.ignore(pos, expected)) { if (!ParserTablesInSelectQuery().parse(pos, tables, expected)) return false; diff --git a/src/Parsers/ParserTablesInSelectQuery.cpp b/src/Parsers/ParserTablesInSelectQuery.cpp index cff4c959267..2247167c66e 100644 --- a/src/Parsers/ParserTablesInSelectQuery.cpp +++ b/src/Parsers/ParserTablesInSelectQuery.cpp @@ -21,9 +21,9 @@ bool ParserTableExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expec { auto res = std::make_shared(); - if (!ParserWithOptionalAlias(std::make_unique(), true).parse(pos, res->subquery, expected) - && !ParserWithOptionalAlias(std::make_unique(false, true), true).parse(pos, res->table_function, expected) - && !ParserWithOptionalAlias(std::make_unique(true, true), true) + if (!ParserWithOptionalAlias(std::make_unique(), allow_alias_without_as_keyword).parse(pos, res->subquery, expected) + && !ParserWithOptionalAlias(std::make_unique(false, true), allow_alias_without_as_keyword).parse(pos, res->table_function, expected) + && !ParserWithOptionalAlias(std::make_unique(true, true), allow_alias_without_as_keyword) .parse(pos, res->database_and_table_name, expected)) return false; @@ -126,7 +126,7 @@ bool ParserTablesInSelectQueryElement::parseImpl(Pos & pos, ASTPtr & node, Expec if (is_first) { - if (!ParserTableExpression().parse(pos, res->table_expression, expected)) + if (!ParserTableExpression(allow_alias_without_as_keyword).parse(pos, res->table_expression, expected)) return false; } else if (ParserArrayJoin().parse(pos, res->array_join, expected)) @@ -200,7 +200,7 @@ bool ParserTablesInSelectQueryElement::parseImpl(Pos & pos, ASTPtr & node, Expec return false; } - if (!ParserTableExpression().parse(pos, res->table_expression, expected)) + if (!ParserTableExpression(allow_alias_without_as_keyword).parse(pos, res->table_expression, expected)) return false; if (table_join->kind != JoinKind::Comma @@ -261,12 +261,12 @@ bool ParserTablesInSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e ASTPtr child; - if (ParserTablesInSelectQueryElement(true).parse(pos, child, expected)) + if (ParserTablesInSelectQueryElement(true, allow_alias_without_as_keyword).parse(pos, child, expected)) res->children.emplace_back(child); else return false; - while (ParserTablesInSelectQueryElement(false).parse(pos, child, expected)) + while (ParserTablesInSelectQueryElement(false, allow_alias_without_as_keyword).parse(pos, child, expected)) res->children.emplace_back(child); node = res; diff --git a/src/Parsers/ParserTablesInSelectQuery.h b/src/Parsers/ParserTablesInSelectQuery.h index 772f1992f4d..428b1482663 100644 --- a/src/Parsers/ParserTablesInSelectQuery.h +++ b/src/Parsers/ParserTablesInSelectQuery.h @@ -12,16 +12,24 @@ struct ASTTableJoin; */ class ParserTablesInSelectQuery : public IParserBase { +public: + explicit ParserTablesInSelectQuery(bool allow_alias_without_as_keyword_ = true) + : allow_alias_without_as_keyword(allow_alias_without_as_keyword_) {} + protected: const char * getName() const override { return "table, table function, subquery or list of joined tables"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +private: + bool allow_alias_without_as_keyword; }; class ParserTablesInSelectQueryElement : public IParserBase { public: - explicit ParserTablesInSelectQueryElement(bool is_first_) : is_first(is_first_) {} + explicit ParserTablesInSelectQueryElement(bool is_first_, bool allow_alias_without_as_keyword_ = true) + : is_first(is_first_), allow_alias_without_as_keyword(allow_alias_without_as_keyword_) {} protected: const char * getName() const override { return "table, table function, subquery or list of joined tables"; } @@ -29,6 +37,7 @@ protected: private: bool is_first; + bool allow_alias_without_as_keyword; static void parseJoinStrictness(Pos & pos, ASTTableJoin & table_join); }; @@ -36,9 +45,16 @@ private: class ParserTableExpression : public IParserBase { +public: + explicit ParserTableExpression(bool allow_alias_without_as_keyword_ = true) + : allow_alias_without_as_keyword(allow_alias_without_as_keyword_) {} + protected: const char * getName() const override { return "table or subquery or table function"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +private: + bool allow_alias_without_as_keyword; }; diff --git a/src/Parsers/fuzzers/create_parser_fuzzer.cpp b/src/Parsers/fuzzers/create_parser_fuzzer.cpp index 032d9ca3ffe..13cb1dfd36e 100644 --- a/src/Parsers/fuzzers/create_parser_fuzzer.cpp +++ b/src/Parsers/fuzzers/create_parser_fuzzer.cpp @@ -15,6 +15,12 @@ try DB::ParserCreateQuery parser; DB::ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 1000); + const UInt64 max_ast_depth = 1000; + ast->checkDepth(max_ast_depth); + + const UInt64 max_ast_elements = 50000; + ast->checkSize(max_ast_elements); + DB::WriteBufferFromOwnString wb; DB::formatAST(*ast, wb); diff --git a/src/Planner/CollectSets.cpp b/src/Planner/CollectSets.cpp index aa7014aba48..e63b3ef078d 100644 --- a/src/Planner/CollectSets.cpp +++ b/src/Planner/CollectSets.cpp @@ -54,12 +54,12 @@ public: { planner_context.registerSet(set_key, PlannerSet(storage_set->getSet())); } - else if (auto constant_value = in_second_argument->getConstantValueOrNull()) + else if (const auto * constant_node = in_second_argument->as()) { auto set = makeSetForConstantValue( in_first_argument->getResultType(), - constant_value->getValue(), - constant_value->getType(), + constant_node->getValue(), + constant_node->getResultType(), settings); planner_context.registerSet(set_key, PlannerSet(std::move(set))); diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 28be1a83088..aec5a578774 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -495,7 +495,8 @@ void Planner::buildQueryPlanIfNeeded() settings.group_by_use_nulls, std::move(input_order_info), std::move(group_by_sort_description), - should_produce_results_in_order_of_bucket_number); + should_produce_results_in_order_of_bucket_number, + settings.enable_memory_bound_merging_of_aggregation_results); query_plan.addStep(std::move(aggregating_step)); if (query_node.isGroupByWithTotals()) @@ -599,7 +600,7 @@ void Planner::buildQueryPlanIfNeeded() if (query_node.hasOffset()) { /// Constness of offset is validated during query analysis stage - limit_offset = query_node.getOffset()->getConstantValue().getValue().safeGet(); + limit_offset = query_node.getOffset()->as().getValue().safeGet(); } UInt64 limit_length = 0; @@ -607,7 +608,7 @@ void Planner::buildQueryPlanIfNeeded() if (query_node.hasLimit()) { /// Constness of limit is validated during query analysis stage - limit_length = query_node.getLimit()->getConstantValue().getValue().safeGet(); + limit_length = query_node.getLimit()->as().getValue().safeGet(); } if (query_node.isDistinct()) @@ -779,13 +780,13 @@ void Planner::buildQueryPlanIfNeeded() query_plan.addStep(std::move(expression_step_before_limit_by)); /// Constness of LIMIT BY limit is validated during query analysis stage - UInt64 limit_by_limit = query_node.getLimitByLimit()->getConstantValue().getValue().safeGet(); + UInt64 limit_by_limit = query_node.getLimitByLimit()->as().getValue().safeGet(); UInt64 limit_by_offset = 0; if (query_node.hasLimitByOffset()) { /// Constness of LIMIT BY offset is validated during query analysis stage - limit_by_offset = query_node.getLimitByOffset()->getConstantValue().getValue().safeGet(); + limit_by_offset = query_node.getLimitByOffset()->as().getValue().safeGet(); } auto limit_by_step = std::make_unique(query_plan.getCurrentDataStream(), diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index a6f1a74f251..aa1b61e5559 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -165,8 +165,6 @@ private: NodeNameAndNodeMinLevel visitColumn(const QueryTreeNodePtr & node); - NodeNameAndNodeMinLevel visitConstantValue(const Field & constant_literal, const DataTypePtr & constant_type); - NodeNameAndNodeMinLevel visitConstant(const QueryTreeNodePtr & node); NodeNameAndNodeMinLevel visitLambda(const QueryTreeNodePtr & node); @@ -175,8 +173,6 @@ private: NodeNameAndNodeMinLevel visitFunction(const QueryTreeNodePtr & node); - NodeNameAndNodeMinLevel visitQueryOrUnion(const QueryTreeNodePtr & node); - std::vector actions_stack; std::unordered_map node_to_node_name; const PlannerContextPtr planner_context; @@ -219,11 +215,9 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi return visitConstant(node); else if (node_type == QueryTreeNodeType::FUNCTION) return visitFunction(node); - else if (node_type == QueryTreeNodeType::QUERY || node_type == QueryTreeNodeType::UNION) - return visitQueryOrUnion(node); throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Expected column, constant, function, query or union node. Actual {}", + "Expected column, constant, function. Actual {}", node->formatASTForErrorMessage()); } @@ -249,8 +243,12 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi return {column_node_name, 0}; } -PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitConstantValue(const Field & constant_literal, const DataTypePtr & constant_type) +PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitConstant(const QueryTreeNodePtr & node) { + const auto & constant_node = node->as(); + const auto & constant_literal = constant_node.getValue(); + const auto & constant_type = constant_node.getResultType(); + auto constant_node_name = calculateConstantActionNodeName(constant_literal, constant_type); ColumnWithTypeAndName column; @@ -268,12 +266,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi } return {constant_node_name, 0}; -} -PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitConstant(const QueryTreeNodePtr & node) -{ - const auto & constant_node = node->as(); - return visitConstantValue(constant_node.getValue(), constant_node.getResultType()); } PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitLambda(const QueryTreeNodePtr & node) @@ -381,11 +374,8 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitFunction(const QueryTreeNodePtr & node) { const auto & function_node = node->as(); - if (const auto constant_value_or_null = function_node.getConstantValueOrNull()) - return visitConstantValue(constant_value_or_null->getValue(), constant_value_or_null->getType()); std::optional in_function_second_argument_node_name_with_level; - if (isNameOfInFunction(function_node.getFunctionName())) in_function_second_argument_node_name_with_level = makeSetForInFunction(node); @@ -466,16 +456,6 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi return {function_node_name, level}; } -PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitQueryOrUnion(const QueryTreeNodePtr & node) -{ - const auto constant_value = node->getConstantValueOrNull(); - if (!constant_value) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Scalar subqueries must be evaluated as constants"); - - return visitConstantValue(constant_value->getValue(), constant_value->getType()); -} - } PlannerActionsVisitor::PlannerActionsVisitor(const PlannerContextPtr & planner_context_) @@ -523,93 +503,71 @@ String calculateActionNodeName(const QueryTreeNodePtr & node, const PlannerConte } case QueryTreeNodeType::FUNCTION: { - if (auto node_constant_value = node->getConstantValueOrNull()) + const auto & function_node = node->as(); + String in_function_second_argument_node_name; + + if (isNameOfInFunction(function_node.getFunctionName())) { - result = calculateConstantActionNodeName(node_constant_value->getValue(), node_constant_value->getType()); + const auto & in_second_argument_node = function_node.getArguments().getNodes().at(1); + in_function_second_argument_node_name = planner_context.createSetKey(in_second_argument_node); } - else + + WriteBufferFromOwnString buffer; + buffer << function_node.getFunctionName(); + + const auto & function_parameters_nodes = function_node.getParameters().getNodes(); + + if (!function_parameters_nodes.empty()) { - const auto & function_node = node->as(); - String in_function_second_argument_node_name; - - if (isNameOfInFunction(function_node.getFunctionName())) - { - const auto & in_second_argument_node = function_node.getArguments().getNodes().at(1); - in_function_second_argument_node_name = planner_context.createSetKey(in_second_argument_node); - } - - WriteBufferFromOwnString buffer; - buffer << function_node.getFunctionName(); - - const auto & function_parameters_nodes = function_node.getParameters().getNodes(); - - if (!function_parameters_nodes.empty()) - { - buffer << '('; - - size_t function_parameters_nodes_size = function_parameters_nodes.size(); - for (size_t i = 0; i < function_parameters_nodes_size; ++i) - { - const auto & function_parameter_node = function_parameters_nodes[i]; - buffer << calculateActionNodeName(function_parameter_node, planner_context, node_to_name); - - if (i + 1 != function_parameters_nodes_size) - buffer << ", "; - } - - buffer << ')'; - } - - const auto & function_arguments_nodes = function_node.getArguments().getNodes(); - String function_argument_name; - buffer << '('; - size_t function_arguments_nodes_size = function_arguments_nodes.size(); - for (size_t i = 0; i < function_arguments_nodes_size; ++i) + size_t function_parameters_nodes_size = function_parameters_nodes.size(); + for (size_t i = 0; i < function_parameters_nodes_size; ++i) { - if (i == 1 && !in_function_second_argument_node_name.empty()) - { - function_argument_name = in_function_second_argument_node_name; - } - else - { - const auto & function_argument_node = function_arguments_nodes[i]; - function_argument_name = calculateActionNodeName(function_argument_node, planner_context, node_to_name); - } + const auto & function_parameter_node = function_parameters_nodes[i]; + buffer << calculateActionNodeName(function_parameter_node, planner_context, node_to_name); - buffer << function_argument_name; - - if (i + 1 != function_arguments_nodes_size) + if (i + 1 != function_parameters_nodes_size) buffer << ", "; } buffer << ')'; + } - if (function_node.isWindowFunction()) + const auto & function_arguments_nodes = function_node.getArguments().getNodes(); + String function_argument_name; + + buffer << '('; + + size_t function_arguments_nodes_size = function_arguments_nodes.size(); + for (size_t i = 0; i < function_arguments_nodes_size; ++i) + { + if (i == 1 && !in_function_second_argument_node_name.empty()) { - buffer << " OVER ("; - buffer << calculateWindowNodeActionName(function_node.getWindowNode(), planner_context, node_to_name); - buffer << ')'; + function_argument_name = in_function_second_argument_node_name; + } + else + { + const auto & function_argument_node = function_arguments_nodes[i]; + function_argument_name = calculateActionNodeName(function_argument_node, planner_context, node_to_name); } - result = buffer.str(); + buffer << function_argument_name; + + if (i + 1 != function_arguments_nodes_size) + buffer << ", "; } - break; - } - case QueryTreeNodeType::UNION: - [[fallthrough]]; - case QueryTreeNodeType::QUERY: - { - if (auto node_constant_value = node->getConstantValueOrNull()) + + buffer << ')'; + + if (function_node.isWindowFunction()) { - result = calculateConstantActionNodeName(node_constant_value->getValue(), node_constant_value->getType()); - } - else - { - auto query_hash = node->getTreeHash(); - result = "__subquery_" + std::to_string(query_hash.first) + '_' + std::to_string(query_hash.second); + buffer << " OVER ("; + buffer << calculateWindowNodeActionName(function_node.getWindowNode(), planner_context, node_to_name); + buffer << ')'; } + + result = buffer.str(); break; } case QueryTreeNodeType::LAMBDA: diff --git a/src/Planner/PlannerAggregation.cpp b/src/Planner/PlannerAggregation.cpp index 3322ef9364f..a1a8b54426a 100644 --- a/src/Planner/PlannerAggregation.cpp +++ b/src/Planner/PlannerAggregation.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -10,6 +11,8 @@ #include +#include + #include namespace DB @@ -203,7 +206,7 @@ AggregateDescriptions extractAggregateDescriptions(const QueryTreeNodes & aggreg for (const auto & parameter_node : parameters_nodes) { /// Function parameters constness validated during analysis stage - aggregate_description.parameters.push_back(parameter_node->getConstantValue().getValue()); + aggregate_description.parameters.push_back(parameter_node->as().getValue()); } const auto & arguments_nodes = aggregate_function_node_typed.getArguments().getNodes(); diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index b034edf97d8..9db268512be 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -96,7 +97,7 @@ std::optional analyzeAggregation(QueryTreeNodePtr & q for (auto & grouping_set_key_node : grouping_set_keys_list_node_typed.getNodes()) { - group_by_with_constant_keys |= grouping_set_key_node->hasConstantValue(); + group_by_with_constant_keys |= (grouping_set_key_node->as() != nullptr); auto expression_dag_nodes = actions_visitor.visit(before_aggregation_actions, grouping_set_key_node); aggregation_keys.reserve(expression_dag_nodes.size()); @@ -147,7 +148,7 @@ std::optional analyzeAggregation(QueryTreeNodePtr & q else { for (auto & group_by_key_node : query_node.getGroupBy().getNodes()) - group_by_with_constant_keys |= group_by_key_node->hasConstantValue(); + group_by_with_constant_keys |= (group_by_key_node->as() != nullptr); auto expression_dag_nodes = actions_visitor.visit(before_aggregation_actions, query_node.getGroupByNode()); aggregation_keys.reserve(expression_dag_nodes.size()); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 0566b579be1..2fd469986ec 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -494,7 +494,8 @@ QueryPlan buildQueryPlanForJoinNode(QueryTreeNodePtr join_tree_node, } } - auto left_table_names = left_plan.getCurrentDataStream().header.getNames(); + const Block & left_header = left_plan.getCurrentDataStream().header; + auto left_table_names = left_header.getNames(); NameSet left_table_names_set(left_table_names.begin(), left_table_names.end()); auto columns_from_joined_table = right_plan.getCurrentDataStream().header.getNamesAndTypesList(); @@ -506,7 +507,8 @@ QueryPlan buildQueryPlanForJoinNode(QueryTreeNodePtr join_tree_node, table_join->addJoinedColumn(column_from_joined_table); } - auto join_algorithm = chooseJoinAlgorithm(table_join, join_node.getRightTableExpression(), right_plan.getCurrentDataStream().header, planner_context); + const Block & right_header = right_plan.getCurrentDataStream().header; + auto join_algorithm = chooseJoinAlgorithm(table_join, join_node.getRightTableExpression(), left_header, right_header, planner_context); auto result_plan = QueryPlan(); diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index b59dccc92c2..019933f9b72 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include #include #include +#include #include #include @@ -293,12 +295,6 @@ JoinClausesAndActions buildJoinClausesAndActions(const ColumnsWithTypeAndName & for (const auto & node : join_expression_actions_nodes) join_expression_dag_input_nodes.insert(&node); - auto * function_node = join_node.getJoinExpression()->as(); - if (!function_node) - throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION, - "JOIN {} join expression expected function", - join_node.formatASTForErrorMessage()); - /** It is possible to have constant value in JOIN ON section, that we need to ignore during DAG construction. * If we do not ignore it, this function will be replaced by underlying constant. * For example ASOF JOIN does not support JOIN with constants, and we should process it like ordinary JOIN. @@ -306,18 +302,25 @@ JoinClausesAndActions buildJoinClausesAndActions(const ColumnsWithTypeAndName & * Example: SELECT * FROM (SELECT 1 AS id, 1 AS value) AS t1 ASOF LEFT JOIN (SELECT 1 AS id, 1 AS value) AS t2 * ON (t1.id = t2.id) AND 1 != 1 AND (t1.value >= t1.value); */ - auto constant_value = function_node->getConstantValueOrNull(); - function_node->performConstantFolding({}); + auto join_expression = join_node.getJoinExpression(); + auto * constant_join_expression = join_expression->as(); + + if (constant_join_expression && constant_join_expression->hasSourceExpression()) + join_expression = constant_join_expression->getSourceExpression(); + + auto * function_node = join_expression->as(); + if (!function_node) + throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION, + "JOIN {} join expression expected function", + join_node.formatASTForErrorMessage()); PlannerActionsVisitor join_expression_visitor(planner_context); - auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(join_expression_actions, join_node.getJoinExpression()); + auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(join_expression_actions, join_expression); if (join_expression_dag_node_raw_pointers.size() != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "JOIN {} ON clause contains multiple expressions", join_node.formatASTForErrorMessage()); - function_node->performConstantFolding(std::move(constant_value)); - const auto * join_expressions_actions_root_node = join_expression_dag_node_raw_pointers[0]; if (!join_expressions_actions_root_node->function) throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION, @@ -540,12 +543,12 @@ std::optional tryExtractConstantFromJoinNode(const QueryTreeNodePtr & join if (!join_node_typed.getJoinExpression()) return {}; - auto constant_value = join_node_typed.getJoinExpression()->getConstantValueOrNull(); - if (!constant_value) + const auto * constant_node = join_node_typed.getJoinExpression()->as(); + if (!constant_node) return {}; - const auto & value = constant_value->getValue(); - auto constant_type = constant_value->getType(); + const auto & value = constant_node->getValue(); + auto constant_type = constant_node->getResultType(); constant_type = removeNullable(removeLowCardinality(constant_type)); auto which_constant_type = WhichDataType(constant_type); @@ -662,6 +665,7 @@ std::shared_ptr tryDirectJoin(const std::shared_ptr chooseJoinAlgorithm(std::shared_ptr & table_join, const QueryTreeNodePtr & right_table_expression, + const Block & left_table_expression_header, const Block & right_table_expression_header, const PlannerContextPtr & planner_context) { @@ -720,6 +724,20 @@ std::shared_ptr chooseJoinAlgorithm(std::shared_ptr & table_jo return std::make_shared(table_join, right_table_expression_header); } + if (table_join->isEnabledAlgorithm(JoinAlgorithm::GRACE_HASH)) + { + if (GraceHashJoin::isSupported(table_join)) + { + auto query_context = planner_context->getQueryContext(); + return std::make_shared( + query_context, + table_join, + left_table_expression_header, + right_table_expression_header, + query_context->getTempDataOnDisk()); + } + } + if (table_join->isEnabledAlgorithm(JoinAlgorithm::AUTO)) return std::make_shared(table_join, right_table_expression_header); diff --git a/src/Planner/PlannerJoins.h b/src/Planner/PlannerJoins.h index d305249e789..c61bce932e0 100644 --- a/src/Planner/PlannerJoins.h +++ b/src/Planner/PlannerJoins.h @@ -190,6 +190,7 @@ std::optional tryExtractConstantFromJoinNode(const QueryTreeNodePtr & join */ std::shared_ptr chooseJoinAlgorithm(std::shared_ptr & table_join, const QueryTreeNodePtr & right_table_expression, + const Block & left_table_expression_header, const Block & right_table_expression_header, const PlannerContextPtr & planner_context); diff --git a/src/Planner/PlannerSorting.cpp b/src/Planner/PlannerSorting.cpp index 5ae8bd1e21b..611a26f78fa 100644 --- a/src/Planner/PlannerSorting.cpp +++ b/src/Planner/PlannerSorting.cpp @@ -24,11 +24,11 @@ namespace std::pair extractWithFillValue(const QueryTreeNodePtr & node) { - const auto & constant_value = node->getConstantValue(); + const auto & constant_node = node->as(); std::pair result; - result.first = constant_value.getValue(); - result.second = constant_value.getType(); + result.first = constant_node.getValue(); + result.second = constant_node.getResultType(); if (!isColumnedAsNumber(result.second)) throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, "WITH FILL expression must be constant with numeric type"); @@ -38,16 +38,16 @@ std::pair extractWithFillValue(const QueryTreeNodePtr & node std::pair> extractWithFillStepValue(const QueryTreeNodePtr & node) { - const auto & constant_value = node->getConstantValue(); + const auto & constant_node = node->as(); - const auto & constant_node_result_type = constant_value.getType(); + const auto & constant_node_result_type = constant_node.getResultType(); if (const auto * type_interval = typeid_cast(constant_node_result_type.get())) - return std::make_pair(constant_value.getValue(), type_interval->getKind()); + return std::make_pair(constant_node.getValue(), type_interval->getKind()); if (!isColumnedAsNumber(constant_node_result_type)) throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, "WITH FILL expression must be constant with numeric type"); - return {constant_value.getValue(), {}}; + return {constant_node.getValue(), {}}; } FillColumnDescription extractWithFillDescription(const SortNode & sort_node) diff --git a/src/Planner/PlannerWindowFunctions.cpp b/src/Planner/PlannerWindowFunctions.cpp index 5f4427d98d4..ce74d82c08d 100644 --- a/src/Planner/PlannerWindowFunctions.cpp +++ b/src/Planner/PlannerWindowFunctions.cpp @@ -1,5 +1,6 @@ #include +#include #include #include @@ -91,7 +92,7 @@ std::vector extractWindowDescriptions(const QueryTreeNodes & for (const auto & parameter_node : parameters_nodes) { /// Function parameters constness validated during analysis stage - window_function.function_parameters.push_back(parameter_node->getConstantValue().getValue()); + window_function.function_parameters.push_back(parameter_node->as().getValue()); } const auto & arguments_nodes = window_function_node_typed.getArguments().getNodes(); diff --git a/src/Processors/Executors/CompletedPipelineExecutor.cpp b/src/Processors/Executors/CompletedPipelineExecutor.cpp index a4c7fe2f687..22b924337c5 100644 --- a/src/Processors/Executors/CompletedPipelineExecutor.cpp +++ b/src/Processors/Executors/CompletedPipelineExecutor.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -33,6 +34,10 @@ struct CompletedPipelineExecutor::Data static void threadFunction(CompletedPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads) { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); setThreadName("QueryCompPipeEx"); try diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 3772381de04..a9083d8c4a8 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -306,6 +306,10 @@ void PipelineExecutor::spawnThreads() { /// ThreadStatus thread_status; + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); setThreadName("QueryPipelineEx"); if (thread_group) diff --git a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp index 596f8e8dedd..5799fbcc5d8 100644 --- a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp @@ -69,6 +69,10 @@ const Block & PullingAsyncPipelineExecutor::getHeader() const static void threadFunction(PullingAsyncPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads) { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); setThreadName("QueryPullPipeEx"); try diff --git a/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp index ee8e94b6f28..54c1e7bf30f 100644 --- a/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp +++ b/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -98,6 +99,10 @@ struct PushingAsyncPipelineExecutor::Data static void threadFunction(PushingAsyncPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads) { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); setThreadName("QueryPushPipeEx"); try diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp index 05fc3b8ca2a..cae4cbab0d7 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp @@ -187,6 +187,7 @@ void registerInputFormatArrow(FormatFactory & factory) { return std::make_shared(buf, sample, false, format_settings); }); + factory.markFormatSupportsSubcolumns("Arrow"); factory.markFormatSupportsSubsetOfColumns("Arrow"); factory.registerInputFormat( "ArrowStream", diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index e9b01ec7dda..8b546f48116 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -324,14 +324,31 @@ static ColumnPtr readOffsetsFromArrowListColumn(std::shared_ptr &>(*offsets_column).getData(); offsets_data.reserve(arrow_column->length()); + uint64_t start_offset = 0u; + for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) { arrow::ListArray & list_chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); auto arrow_offsets_array = list_chunk.offsets(); auto & arrow_offsets = dynamic_cast(*arrow_offsets_array); - auto start = offsets_data.back(); + + /* + * It seems like arrow::ListArray::values() (nested column data) might or might not be shared across chunks. + * When it is shared, the offsets will be monotonically increasing. Otherwise, the offsets will be zero based. + * In order to account for both cases, the starting offset is updated whenever a zero-based offset is found. + * More info can be found in: https://lists.apache.org/thread/rrwfb9zo2dc58dhd9rblf20xd7wmy7jm and + * https://github.com/ClickHouse/ClickHouse/pull/43297 + * */ + if (list_chunk.offset() == 0) + { + start_offset = offsets_data.back(); + } + for (int64_t i = 1; i < arrow_offsets.length(); ++i) - offsets_data.emplace_back(start + arrow_offsets.Value(i)); + { + auto offset = arrow_offsets.Value(i); + offsets_data.emplace_back(start_offset + offset); + } } return offsets_column; } @@ -467,8 +484,23 @@ static std::shared_ptr getNestedArrowColumn(std::shared_ptr for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) { arrow::ListArray & list_chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); - std::shared_ptr chunk = list_chunk.values(); - array_vector.emplace_back(std::move(chunk)); + + /* + * It seems like arrow::ListArray::values() (nested column data) might or might not be shared across chunks. + * Therefore, simply appending arrow::ListArray::values() could lead to duplicated data to be appended. + * To properly handle this, arrow::ListArray::values() needs to be sliced based on the chunk offsets. + * arrow::ListArray::Flatten does that. More info on: https://lists.apache.org/thread/rrwfb9zo2dc58dhd9rblf20xd7wmy7jm and + * https://github.com/ClickHouse/ClickHouse/pull/43297 + * */ + auto flatten_result = list_chunk.Flatten(); + if (flatten_result.ok()) + { + array_vector.emplace_back(flatten_result.ValueOrDie()); + } + else + { + throw Exception(ErrorCodes::INCORRECT_DATA, "Failed to flatten chunk '{}' of column of type '{}' ", chunk_i, arrow_column->type()->id()); + } } return std::make_shared(array_vector); } diff --git a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp index 047a55d3f90..a41cf687b39 100644 --- a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp @@ -59,7 +59,7 @@ std::vector BinaryFormatReader::readTypes() bool BinaryFormatReader::readField(IColumn & column, const DataTypePtr & /*type*/, const SerializationPtr & serialization, bool /*is_last_file_column*/, const String & /*column_name*/) { - serialization->deserializeBinary(column, *in); + serialization->deserializeBinary(column, *in, format_settings); return true; } @@ -92,7 +92,7 @@ void BinaryFormatReader::skipField(size_t file_column) if (file_column >= read_data_types.size()) throw Exception(ErrorCodes::CANNOT_SKIP_UNKNOWN_FIELD, "Cannot skip unknown field in RowBinaryWithNames format, because it's type is unknown"); Field field; - read_data_types[file_column]->getDefaultSerialization()->deserializeBinary(field, *in); + read_data_types[file_column]->getDefaultSerialization()->deserializeBinary(field, *in, format_settings); } BinaryWithNamesAndTypesSchemaReader::BinaryWithNamesAndTypesSchemaReader(ReadBuffer & in_, const FormatSettings & format_settings_) diff --git a/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp b/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp index 60b722569a2..c9ed8e03449 100644 --- a/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp @@ -10,8 +10,8 @@ namespace DB { -BinaryRowOutputFormat::BinaryRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names_, bool with_types_, const RowOutputFormatParams & params_) - : IRowOutputFormat(header, out_, params_), with_names(with_names_), with_types(with_types_) +BinaryRowOutputFormat::BinaryRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names_, bool with_types_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_) + : IRowOutputFormat(header, out_, params_), with_names(with_names_), with_types(with_types_), format_settings(format_settings_) { } @@ -44,7 +44,7 @@ void BinaryRowOutputFormat::writePrefix() void BinaryRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) { - serialization.serializeBinary(column, row_num, out); + serialization.serializeBinary(column, row_num, out, format_settings); } @@ -56,9 +56,9 @@ void registerOutputFormatRowBinary(FormatFactory & factory) WriteBuffer & buf, const Block & sample, const RowOutputFormatParams & params, - const FormatSettings &) + const FormatSettings & format_settings) { - return std::make_shared(buf, sample, with_names, with_types, params); + return std::make_shared(buf, sample, with_names, with_types, params, format_settings); }); factory.markOutputFormatSupportsParallelFormatting(format_name); }; diff --git a/src/Processors/Formats/Impl/BinaryRowOutputFormat.h b/src/Processors/Formats/Impl/BinaryRowOutputFormat.h index 40894608677..e8198cb6ee0 100644 --- a/src/Processors/Formats/Impl/BinaryRowOutputFormat.h +++ b/src/Processors/Formats/Impl/BinaryRowOutputFormat.h @@ -17,7 +17,7 @@ class WriteBuffer; class BinaryRowOutputFormat final: public IRowOutputFormat { public: - BinaryRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names_, bool with_types_, const RowOutputFormatParams & params_); + BinaryRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names_, bool with_types_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_); String getName() const override { return "BinaryRowOutputFormat"; } @@ -29,6 +29,7 @@ private: bool with_names; bool with_types; + const FormatSettings format_settings; }; } diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index d6dbd69135a..58fd03a7a78 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -198,6 +198,7 @@ void registerInputFormatORC(FormatFactory & factory) { return std::make_shared(buf, sample, settings); }); + factory.markFormatSupportsSubcolumns("ORC"); factory.markFormatSupportsSubsetOfColumns("ORC"); } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp index 32ab391cf8c..40ab6554115 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp @@ -1,6 +1,7 @@ #include #include +#include namespace DB { @@ -97,6 +98,10 @@ namespace DB void ParallelFormattingOutputFormat::collectorThreadFunction(const ThreadGroupStatusPtr & thread_group) { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); setThreadName("Collector"); if (thread_group) CurrentThread::attachToIfDetached(thread_group); @@ -154,6 +159,10 @@ namespace DB void ParallelFormattingOutputFormat::formatterThreadFunction(size_t current_unit_number, size_t first_row_num, const ThreadGroupStatusPtr & thread_group) { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); setThreadName("Formatter"); if (thread_group) CurrentThread::attachToIfDetached(thread_group); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 9172c79c890..19ec3772da0 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -3,12 +3,17 @@ #include #include #include +#include namespace DB { void ParallelParsingInputFormat::segmentatorThreadFunction(ThreadGroupStatusPtr thread_group) { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); if (thread_group) CurrentThread::attachTo(thread_group); @@ -55,6 +60,10 @@ void ParallelParsingInputFormat::segmentatorThreadFunction(ThreadGroupStatusPtr void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr thread_group, size_t current_ticket_number) { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); if (thread_group) CurrentThread::attachToIfDetached(thread_group); diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index dd2826287b2..c2253fe4b20 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -201,6 +201,7 @@ void registerInputFormatParquet(FormatFactory & factory) { return std::make_shared(buf, sample, settings); }); + factory.markFormatSupportsSubcolumns("Parquet"); factory.markFormatSupportsSubsetOfColumns("Parquet"); } diff --git a/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp b/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp index 19eeec979c7..aef1e9c70da 100644 --- a/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp @@ -55,11 +55,12 @@ void FinishAggregatingInOrderAlgorithm::consume(Input & input, size_t source_num if (!info) throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk info was not set for chunk in FinishAggregatingInOrderAlgorithm"); - const auto * arenas_info = typeid_cast(info.get()); - if (!arenas_info) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk should have ChunkInfoWithAllocatedBytes in FinishAggregatingInOrderAlgorithm"); + Int64 allocated_bytes = 0; + /// Will be set by AggregatingInOrderTransform during local aggregation; will be nullptr during merging on initiator. + if (const auto * arenas_info = typeid_cast(info.get())) + allocated_bytes = arenas_info->allocated_bytes; - states[source_num] = State{input.chunk, description, arenas_info->allocated_bytes}; + states[source_num] = State{input.chunk, description, allocated_bytes}; } IMergingAlgorithm::Status FinishAggregatingInOrderAlgorithm::merge() @@ -130,6 +131,7 @@ Chunk FinishAggregatingInOrderAlgorithm::prepareToMerge() auto info = std::make_shared(); info->chunks = std::make_unique(std::move(chunks)); + info->chunk_num = chunk_num++; Chunk chunk; chunk.setChunkInfo(std::move(info)); diff --git a/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.h b/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.h index ff31886f438..b1a74a09459 100644 --- a/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.h +++ b/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.h @@ -88,6 +88,7 @@ private: std::vector inputs_to_update; std::vector chunks; + UInt64 chunk_num = 0; size_t accumulated_rows = 0; size_t accumulated_bytes = 0; }; diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index e89392d2e1f..8ef547ee8ab 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -1,34 +1,45 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include +#include #include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include namespace DB { -static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_of_bucket_number) +static bool memoryBoundMergingWillBeUsed( + bool should_produce_results_in_order_of_bucket_number, + bool memory_bound_merging_of_aggregation_results_enabled, + InputOrderInfoPtr group_by_info) +{ + return should_produce_results_in_order_of_bucket_number && memory_bound_merging_of_aggregation_results_enabled && group_by_info; +} + +static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_of_bucket_number, bool memory_bound_merging_will_be_used) { return ITransformingStep::Traits { { .preserves_distinct_columns = false, /// Actually, we may check that distinct names are in aggregation keys - .returns_single_stream = should_produce_results_in_order_of_bucket_number, /// Actually, may also return single stream if should_produce_results_in_order_of_bucket_number = false + .returns_single_stream = should_produce_results_in_order_of_bucket_number || memory_bound_merging_will_be_used, .preserves_number_of_streams = false, .preserves_sorting = false, + .can_enforce_sorting_properties_in_distributed_query = memory_bound_merging_will_be_used, }, { .preserves_number_of_rows = false, @@ -88,9 +99,16 @@ AggregatingStep::AggregatingStep( bool group_by_use_nulls_, InputOrderInfoPtr group_by_info_, SortDescription group_by_sort_description_, - bool should_produce_results_in_order_of_bucket_number_) + bool should_produce_results_in_order_of_bucket_number_, + bool memory_bound_merging_of_aggregation_results_enabled_) : ITransformingStep( - input_stream_, appendGroupingColumn(params_.getHeader(input_stream_.header, final_), params_.keys, grouping_sets_params_, group_by_use_nulls_), getTraits(should_produce_results_in_order_of_bucket_number_), false) + input_stream_, + appendGroupingColumn(params_.getHeader(input_stream_.header, final_), params_.keys, grouping_sets_params_, group_by_use_nulls_), + getTraits( + should_produce_results_in_order_of_bucket_number_, + DB::memoryBoundMergingWillBeUsed( + should_produce_results_in_order_of_bucket_number_, memory_bound_merging_of_aggregation_results_enabled_, group_by_info_)), + false) , params(std::move(params_)) , grouping_sets_params(std::move(grouping_sets_params_)) , final(final_) @@ -103,7 +121,13 @@ AggregatingStep::AggregatingStep( , group_by_info(std::move(group_by_info_)) , group_by_sort_description(std::move(group_by_sort_description_)) , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) + , memory_bound_merging_of_aggregation_results_enabled(memory_bound_merging_of_aggregation_results_enabled_) { + if (memoryBoundMergingWillBeUsed()) + { + output_stream->sort_description = group_by_sort_description; + output_stream->sort_scope = DataStream::SortScope::Global; + } } void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) @@ -336,10 +360,16 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B /// Do merge of aggregated data in parallel. pipeline.resize(merge_threads); - pipeline.addSimpleTransform([&](const Block &) + const auto & required_sort_description = memoryBoundMergingWillBeUsed() ? group_by_sort_description : SortDescription{}; + pipeline.addSimpleTransform( + [&](const Block &) + { return std::make_shared(transform_params, required_sort_description); }); + + if (memoryBoundMergingWillBeUsed()) { - return std::make_shared(transform_params); - }); + pipeline.addTransform( + std::make_shared(pipeline.getHeader(), pipeline.getNumStreams())); + } aggregating_sorted = collector.detachProcessors(1); } @@ -380,7 +410,6 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B return std::make_shared(header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads); }); - /// We add the explicit resize here, but not in case of aggregating in order, since AIO don't use two-level hash tables and thus returns only buckets with bucket_number = -1. pipeline.resize(should_produce_results_in_order_of_bucket_number ? 1 : params.max_threads, true /* force */); aggregating = collector.detachProcessors(0); @@ -426,4 +455,17 @@ void AggregatingStep::updateOutputStream() getDataStreamTraits()); } +void AggregatingStep::adjustSettingsToEnforceSortingPropertiesInDistributedQuery(ContextMutablePtr context) const +{ + context->setSetting("enable_memory_bound_merging_of_aggregation_results", true); + context->setSetting("optimize_aggregation_in_order", true); + context->setSetting("force_aggregation_in_order", true); +} + +bool AggregatingStep::memoryBoundMergingWillBeUsed() const +{ + return DB::memoryBoundMergingWillBeUsed( + should_produce_results_in_order_of_bucket_number, memory_bound_merging_of_aggregation_results_enabled, group_by_info); +} + } diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 71130b65adb..84c6610e90d 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -39,7 +39,8 @@ public: bool group_by_use_nulls_, InputOrderInfoPtr group_by_info_, SortDescription group_by_sort_description_, - bool should_produce_results_in_order_of_bucket_number_); + bool should_produce_results_in_order_of_bucket_number_, + bool memory_bound_merging_of_aggregation_results_enabled_); String getName() const override { return "Aggregating"; } @@ -52,9 +53,13 @@ public: const Aggregator::Params & getParams() const { return params; } + void adjustSettingsToEnforceSortingPropertiesInDistributedQuery(ContextMutablePtr context) const override; + private: void updateOutputStream() override; + bool memoryBoundMergingWillBeUsed() const; + Aggregator::Params params; GroupingSetsParamsList grouping_sets_params; bool final; @@ -69,9 +74,9 @@ private: InputOrderInfoPtr group_by_info; SortDescription group_by_sort_description; - /// It determines if we should resize pipeline to 1 at the end. - /// Needed in case of distributed memory efficient aggregation. - const bool should_produce_results_in_order_of_bucket_number; + /// These settings are used to determine if we should resize pipeline to 1 at the end. + bool should_produce_results_in_order_of_bucket_number; + bool memory_bound_merging_of_aggregation_results_enabled; Processors aggregating_in_order; Processors aggregating_sorted; diff --git a/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.h b/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.h index 8c2eef00af0..b363991c2f6 100644 --- a/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.h +++ b/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include namespace DB diff --git a/src/Processors/QueryPlan/IQueryPlanStep.h b/src/Processors/QueryPlan/IQueryPlanStep.h index c5bd64d66be..1e00d76b66f 100644 --- a/src/Processors/QueryPlan/IQueryPlanStep.h +++ b/src/Processors/QueryPlan/IQueryPlanStep.h @@ -31,13 +31,13 @@ public: /// QueryPipeline has single port. Totals or extremes ports are not counted. bool has_single_port = false; - /// Sorting scope + /// Sorting scope. Please keep the mutual order (more strong mode should have greater value). enum class SortScope { - None, - Chunk, /// Separate chunks are sorted - Stream, /// Each data steam is sorted - Global, /// Data is globally sorted + None = 0, + Chunk = 1, /// Separate chunks are sorted + Stream = 2, /// Each data steam is sorted + Global = 3, /// Data is globally sorted }; /// It is not guaranteed that header has columns from sort_description. diff --git a/src/Processors/QueryPlan/ITransformingStep.h b/src/Processors/QueryPlan/ITransformingStep.h index 008642c71ee..a4124dda806 100644 --- a/src/Processors/QueryPlan/ITransformingStep.h +++ b/src/Processors/QueryPlan/ITransformingStep.h @@ -4,6 +4,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + /// Step which has single input and single output data stream. /// It doesn't mean that pipeline has single port before or after such step. class ITransformingStep : public IQueryPlanStep @@ -29,6 +34,9 @@ public: /// Doesn't change row order. /// Examples: true for FilterStep, false for PartialSortingStep bool preserves_sorting; + + /// See adjustSettingsToEnforceSortingPropertiesInDistributedQuery(). + bool can_enforce_sorting_properties_in_distributed_query = false; }; /// This flags are used by QueryPlan optimizers. @@ -73,6 +81,13 @@ public: /// Append extra processors for this step. void appendExtraProcessors(const Processors & extra_processors); + /// Enforcement is supposed to be done through the special settings that will be taken into account by remote nodes during query planning (e.g. force_aggregation_in_order). + /// Should be called only if data_stream_traits.can_enforce_sorting_properties_in_distributed_query == true. + virtual void adjustSettingsToEnforceSortingPropertiesInDistributedQuery(ContextMutablePtr) const + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented"); + } + protected: /// Clear distinct_columns if res_header doesn't contain all of them. static void updateDistinctColumns(const Block & res_header, NameSet & distinct_columns); diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index d74a6174f00..10b986579cc 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -1,13 +1,25 @@ +#include +#include #include -#include #include -#include +#include #include +#include +#include namespace DB { -static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_of_bucket_number) +static bool memoryBoundMergingWillBeUsed( + const DataStream & input_stream, + bool memory_bound_merging_of_aggregation_results_enabled, + const SortDescription & group_by_sort_description) +{ + return memory_bound_merging_of_aggregation_results_enabled && !group_by_sort_description.empty() + && input_stream.sort_scope >= DataStream::SortScope::Stream && input_stream.sort_description.hasPrefix(group_by_sort_description); +} + +static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_of_bucket_number, bool memory_bound_merging_will_be_used) { return ITransformingStep::Traits { @@ -16,6 +28,7 @@ static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_ .returns_single_stream = should_produce_results_in_order_of_bucket_number, .preserves_number_of_streams = false, .preserves_sorting = false, + .can_enforce_sorting_properties_in_distributed_query = memory_bound_merging_will_be_used, }, { .preserves_number_of_rows = false, @@ -30,24 +43,74 @@ MergingAggregatedStep::MergingAggregatedStep( bool memory_efficient_aggregation_, size_t max_threads_, size_t memory_efficient_merge_threads_, - bool should_produce_results_in_order_of_bucket_number_) + bool should_produce_results_in_order_of_bucket_number_, + size_t max_block_size_, + size_t memory_bound_merging_max_block_bytes_, + SortDescription group_by_sort_description_, + bool memory_bound_merging_of_aggregation_results_enabled_) : ITransformingStep( - input_stream_, params_.getHeader(input_stream_.header, final_), getTraits(should_produce_results_in_order_of_bucket_number_)) + input_stream_, + params_.getHeader(input_stream_.header, final_), + getTraits( + should_produce_results_in_order_of_bucket_number_, + DB::memoryBoundMergingWillBeUsed( + input_stream_, memory_bound_merging_of_aggregation_results_enabled_, group_by_sort_description_))) , params(std::move(params_)) , final(final_) , memory_efficient_aggregation(memory_efficient_aggregation_) , max_threads(max_threads_) , memory_efficient_merge_threads(memory_efficient_merge_threads_) + , max_block_size(max_block_size_) + , memory_bound_merging_max_block_bytes(memory_bound_merging_max_block_bytes_) + , group_by_sort_description(std::move(group_by_sort_description_)) , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) + , memory_bound_merging_of_aggregation_results_enabled(memory_bound_merging_of_aggregation_results_enabled_) { /// Aggregation keys are distinct for (const auto & key : params.keys) output_stream->distinct_columns.insert(key); + + if (memoryBoundMergingWillBeUsed() && should_produce_results_in_order_of_bucket_number) + { + output_stream->sort_description = group_by_sort_description; + output_stream->sort_scope = DataStream::SortScope::Global; + } } void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { auto transform_params = std::make_shared(pipeline.getHeader(), std::move(params), final); + + if (memoryBoundMergingWillBeUsed()) + { + auto transform = std::make_shared( + pipeline.getHeader(), + pipeline.getNumStreams(), + transform_params, + group_by_sort_description, + max_block_size, + memory_bound_merging_max_block_bytes); + + pipeline.addTransform(std::move(transform)); + + /// Do merge of aggregated data in parallel. + pipeline.resize(max_threads); + + const auto & required_sort_description + = should_produce_results_in_order_of_bucket_number ? group_by_sort_description : SortDescription{}; + + pipeline.addSimpleTransform( + [&](const Block &) { return std::make_shared(transform_params, required_sort_description); }); + + if (should_produce_results_in_order_of_bucket_number) + { + pipeline.addTransform( + std::make_shared(pipeline.getHeader(), pipeline.getNumStreams())); + } + + return; + } + if (!memory_efficient_aggregation) { /// We union several sources into one, paralleling the work. @@ -88,5 +151,14 @@ void MergingAggregatedStep::updateOutputStream() output_stream->distinct_columns.insert(key); } - +void MergingAggregatedStep::adjustSettingsToEnforceSortingPropertiesInDistributedQuery(ContextMutablePtr context) const +{ + context->setSetting("enable_memory_bound_merging_of_aggregation_results", true); +} + +bool MergingAggregatedStep::memoryBoundMergingWillBeUsed() const +{ + return DB::memoryBoundMergingWillBeUsed( + input_streams.front(), memory_bound_merging_of_aggregation_results_enabled, group_by_sort_description); +} } diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.h b/src/Processors/QueryPlan/MergingAggregatedStep.h index 419b43615bd..24bf6cfdd2b 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.h +++ b/src/Processors/QueryPlan/MergingAggregatedStep.h @@ -20,7 +20,11 @@ public: bool memory_efficient_aggregation_, size_t max_threads_, size_t memory_efficient_merge_threads_, - bool should_produce_results_in_order_of_bucket_number_); + bool should_produce_results_in_order_of_bucket_number_, + size_t max_block_size_, + size_t memory_bound_merging_max_block_bytes_, + SortDescription group_by_sort_description_, + bool memory_bound_merging_of_aggregation_results_enabled_); String getName() const override { return "MergingAggregated"; } @@ -29,18 +33,25 @@ public: void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; + void adjustSettingsToEnforceSortingPropertiesInDistributedQuery(ContextMutablePtr context) const override; + private: void updateOutputStream() override; + bool memoryBoundMergingWillBeUsed() const; + Aggregator::Params params; bool final; bool memory_efficient_aggregation; size_t max_threads; size_t memory_efficient_merge_threads; + const size_t max_block_size; + const size_t memory_bound_merging_max_block_bytes; + const SortDescription group_by_sort_description; - /// It determines if we should resize pipeline to 1 at the end. - /// Needed in case of distributed memory efficient aggregation over distributed table. + /// These settings are used to determine if we should resize pipeline to 1 at the end. const bool should_produce_results_in_order_of_bucket_number; + const bool memory_bound_merging_of_aggregation_results_enabled; }; } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 59f3e094cb7..a08106054fa 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -64,6 +65,8 @@ static MergeTreeReaderSettings getMergeTreeReaderSettings( .checksum_on_read = settings.checksum_on_read, .read_in_order = query_info.input_order_info != nullptr, .apply_deleted_mask = context->applyDeletedMask(), + .use_asynchronous_read_from_pool = settings.allow_asynchronous_read_from_io_pool_for_merge_tree + && (settings.max_streams_to_max_threads_ratio > 1 || settings.allow_asynchronous_read_from_io_pool_for_merge_tree), }; } @@ -88,7 +91,7 @@ ReadFromMergeTree::ReadFromMergeTree( Poco::Logger * log_, MergeTreeDataSelectAnalysisResultPtr analyzed_result_ptr_, bool enable_parallel_reading) - : ISourceStep(DataStream{.header = MergeTreeBaseSelectProcessor::transformHeader( + : ISourceStep(DataStream{.header = IMergeTreeSelectAlgorithm::transformHeader( storage_snapshot_->getSampleBlockForColumns(real_column_names_), getPrewhereInfoFromQueryInfo(query_info_), data_.getPartitionValueType(), @@ -124,6 +127,21 @@ ReadFromMergeTree::ReadFromMergeTree( if (enable_parallel_reading) read_task_callback = context->getMergeTreeReadTaskCallback(); + const auto & settings = context->getSettingsRef(); + if (settings.max_streams_for_merge_tree_reading) + { + if (settings.allow_asynchronous_read_from_io_pool_for_merge_tree) + { + /// When async reading is enabled, allow to read using more streams. + /// Will add resize to output_streams_limit to reduce memory usage. + output_streams_limit = std::min(requested_num_streams, settings.max_streams_for_merge_tree_reading); + requested_num_streams = std::max(requested_num_streams, settings.max_streams_for_merge_tree_reading); + } + else + /// Just limit requested_num_streams otherwise. + requested_num_streams = std::min(requested_num_streams, settings.max_streams_for_merge_tree_reading); + } + /// Add explicit description. setStepDescription(data.getStorageID().getFullNameNotQuoted()); @@ -210,12 +228,14 @@ Pipe ReadFromMergeTree::readFromPool( }; } - auto source = std::make_shared( + auto algorithm = std::make_unique( i, pool, min_marks_for_concurrent_read, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, data, storage_snapshot, use_uncompressed_cache, prewhere_info, actions_settings, reader_settings, virt_column_names, std::move(extension)); + auto source = std::make_shared(std::move(algorithm)); + /// Set the approximate number of rows for the first source only /// In case of parallel processing on replicas do not set approximate rows at all. /// Because the value will be identical on every replicas and will be accounted @@ -223,13 +243,17 @@ Pipe ReadFromMergeTree::readFromPool( if (i == 0 && !client_info.collaborate_with_initiator) source->addTotalRowsApprox(total_rows); + pipes.emplace_back(std::move(source)); } - return Pipe::unitePipes(std::move(pipes)); + auto pipe = Pipe::unitePipes(std::move(pipes)); + if (output_streams_limit && output_streams_limit < pipe.numOutputPorts()) + pipe.resize(output_streams_limit); + return pipe; } -template +template ProcessorPtr ReadFromMergeTree::createSource( const RangesInDataPart & part, const Names & required_columns, @@ -260,13 +284,15 @@ ProcessorPtr ReadFromMergeTree::createSource( /// because we don't know actual amount of read rows in case when limit is set. bool set_rows_approx = !extension.has_value() && !reader_settings.read_in_order; - auto source = std::make_shared( + auto algorithm = std::make_unique( data, storage_snapshot, part.data_part, max_block_size, preferred_block_size_bytes, preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, prewhere_info, actions_settings, reader_settings, virt_column_names, part.part_index_in_query, has_limit_below_one_block, std::move(extension)); + auto source = std::make_shared(std::move(algorithm)); + if (set_rows_approx) - source -> addTotalRowsApprox(total_rows); + source->addTotalRowsApprox(total_rows); return source; } @@ -286,8 +312,8 @@ Pipe ReadFromMergeTree::readInOrder( for (const auto & part : parts_with_range) { auto source = read_type == ReadType::InReverseOrder - ? createSource(part, required_columns, use_uncompressed_cache, has_limit_below_one_block) - : createSource(part, required_columns, use_uncompressed_cache, has_limit_below_one_block); + ? createSource(part, required_columns, use_uncompressed_cache, has_limit_below_one_block) + : createSource(part, required_columns, use_uncompressed_cache, has_limit_below_one_block); pipes.emplace_back(std::move(source)); } @@ -1088,6 +1114,11 @@ void ReadFromMergeTree::requestReadingInOrder(size_t prefix_size, int direction, reader_settings.read_in_order = true; + /// In case or read-in-order, don't create too many reading streams. + /// Almost always we are reading from a single stream at a time because of merge sort. + if (output_streams_limit) + requested_num_streams = output_streams_limit; + /// update sort info for output stream SortDescription sort_description; const Names & sorting_key_columns = storage_snapshot->getMetadataForQuery()->getSortingKeyColumns(); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 808e849fe03..09edb88eeb8 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -184,7 +184,8 @@ private: ContextPtr context; const size_t max_block_size; - const size_t requested_num_streams; + size_t requested_num_streams; + size_t output_streams_limit = 0; const size_t preferred_block_size_bytes; const size_t preferred_max_column_in_block_size_bytes; const bool sample_factor_column_queried; diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 65b902230f4..81f2fa4b65f 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -76,7 +76,9 @@ ReadFromRemote::ReadFromRemote( Tables external_tables_, Poco::Logger * log_, UInt32 shard_count_, - std::shared_ptr storage_limits_) + std::shared_ptr storage_limits_, + SortDescription output_sort_description_, + DataStream::SortScope output_sort_scope_) : ISourceStep(DataStream{.header = std::move(header_)}) , shards(std::move(shards_)) , stage(stage_) @@ -90,6 +92,8 @@ ReadFromRemote::ReadFromRemote( , log(log_) , shard_count(shard_count_) { + output_stream->sort_description = std::move(output_sort_description_); + output_stream->sort_scope = output_sort_scope_; } void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard) @@ -239,7 +243,9 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( Scalars scalars_, Tables external_tables_, Poco::Logger * log_, - std::shared_ptr storage_limits_) + std::shared_ptr storage_limits_, + SortDescription output_sort_description_, + DataStream::SortScope output_sort_scope_) : ISourceStep(DataStream{.header = std::move(header_)}) , coordinator(std::move(coordinator_)) , shard(std::move(shard_)) @@ -260,6 +266,9 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( description.push_back(fmt::format("Replica: {}", address.host_name)); setStepDescription(boost::algorithm::join(description, ", ")); + + output_stream->sort_description = std::move(output_sort_description_); + output_stream->sort_scope = output_sort_scope_; } diff --git a/src/Processors/QueryPlan/ReadFromRemote.h b/src/Processors/QueryPlan/ReadFromRemote.h index 4d37a637250..7c8bbddfe79 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.h +++ b/src/Processors/QueryPlan/ReadFromRemote.h @@ -33,7 +33,9 @@ public: Tables external_tables_, Poco::Logger * log_, UInt32 shard_count_, - std::shared_ptr storage_limits_); + std::shared_ptr storage_limits_, + SortDescription output_sort_description_, + DataStream::SortScope output_sort_scope_); String getName() const override { return "ReadFromRemote"; } @@ -83,7 +85,9 @@ public: Scalars scalars_, Tables external_tables_, Poco::Logger * log_, - std::shared_ptr storage_limits_); + std::shared_ptr storage_limits_, + SortDescription output_sort_description_, + DataStream::SortScope output_sort_scope_); String getName() const override { return "ReadFromRemoteParallelReplicas"; } diff --git a/src/Processors/QueryPlan/UnionStep.cpp b/src/Processors/QueryPlan/UnionStep.cpp index 5d40a9e241e..6c990c5fd0b 100644 --- a/src/Processors/QueryPlan/UnionStep.cpp +++ b/src/Processors/QueryPlan/UnionStep.cpp @@ -1,8 +1,9 @@ +#include +#include #include -#include #include #include -#include +#include #include namespace DB @@ -35,6 +36,22 @@ UnionStep::UnionStep(DataStreams input_streams_, size_t max_threads_) output_stream = input_streams.front(); else output_stream = DataStream{.header = header}; + + SortDescription common_sort_description = input_streams.front().sort_description; + DataStream::SortScope sort_scope = input_streams.front().sort_scope; + for (const auto & input_stream : input_streams) + { + common_sort_description = commonPrefix(common_sort_description, input_stream.sort_description); + sort_scope = std::min(sort_scope, input_stream.sort_scope); + } + if (!common_sort_description.empty() && sort_scope >= DataStream::SortScope::Chunk) + { + output_stream->sort_description = common_sort_description; + if (sort_scope == DataStream::SortScope::Global && input_streams.size() > 1) + output_stream->sort_scope = DataStream::SortScope::Stream; + else + output_stream->sort_scope = sort_scope; + } } QueryPipelineBuilderPtr UnionStep::updatePipeline(QueryPipelineBuilders pipelines, const BuildQueryPipelineSettings &) diff --git a/src/Processors/Sources/WaitForAsyncInsertSource.h b/src/Processors/Sources/WaitForAsyncInsertSource.h index 40871a59125..1029c164941 100644 --- a/src/Processors/Sources/WaitForAsyncInsertSource.h +++ b/src/Processors/Sources/WaitForAsyncInsertSource.h @@ -6,18 +6,24 @@ namespace DB { +namespace ErrorCodes +{ + extern const int TIMEOUT_EXCEEDED; + extern const int LOGICAL_ERROR; +} + /// Source, that allow to wait until processing of /// asynchronous insert for specified query_id will be finished. class WaitForAsyncInsertSource : public ISource, WithContext { public: WaitForAsyncInsertSource( - const String & query_id_, size_t timeout_ms_, AsynchronousInsertQueue & queue_) + std::future insert_future_, size_t timeout_ms_) : ISource(Block()) - , query_id(query_id_) + , insert_future(std::move(insert_future_)) , timeout_ms(timeout_ms_) - , queue(queue_) { + assert(insert_future.valid()); } String getName() const override { return "WaitForAsyncInsert"; } @@ -25,14 +31,20 @@ public: protected: Chunk generate() override { - queue.waitForProcessingQuery(query_id, std::chrono::milliseconds(timeout_ms)); + auto status = insert_future.wait_for(std::chrono::milliseconds(timeout_ms)); + if (status == std::future_status::deferred) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Logical error: got future in deferred state"); + + if (status == std::future_status::timeout) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Wait for async insert timeout ({} ms) exceeded)", timeout_ms); + + insert_future.get(); return Chunk(); } private: - String query_id; + std::future insert_future; size_t timeout_ms; - AsynchronousInsertQueue & queue; }; } diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index c2de0c3a23a..4664dcae8dd 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -170,7 +170,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk) } } - current_memory_usage = getCurrentMemoryUsage() - initial_memory_usage; + current_memory_usage = std::max(getCurrentMemoryUsage() - initial_memory_usage, 0); /// We finalize last key aggregation state if a new key found. if (key_end != rows) diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 789fa970ebd..0771761fa5c 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -14,6 +15,7 @@ class AggregatedChunkInfo : public ChunkInfo public: bool is_overflows = false; Int32 bucket_num = -1; + UInt64 chunk_num = 0; // chunk number in order of generation, used during memory bound merging to restore chunks order }; using AggregatorList = std::list; @@ -96,6 +98,10 @@ struct ManyAggregatedData pool->trySchedule( [variant = std::move(variant), thread_group = CurrentThread::getGroup()]() { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); if (thread_group) CurrentThread::attachToIfDetached(thread_group); diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index fed28a11ad5..c28a84e9d5d 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -16,6 +16,7 @@ Block JoiningTransform::transformHeader(Block header, const JoinPtr & join) { LOG_DEBUG(&Poco::Logger::get("JoiningTransform"), "Before join block: '{}'", header.dumpStructure()); join->checkTypesOfKeys(header); + join->initialize(header); ExtraBlockPtr tmp; join->joinBlock(header, tmp); LOG_DEBUG(&Poco::Logger::get("JoiningTransform"), "After join block: '{}'", header.dumpStructure()); @@ -38,17 +39,27 @@ JoiningTransform::JoiningTransform( , max_block_size(max_block_size_) { if (!join->isFilled()) - inputs.emplace_back(Block(), this); + inputs.emplace_back(Block(), this); // Wait for FillingRightJoinSideTransform +} + +JoiningTransform::~JoiningTransform() = default; + +OutputPort & JoiningTransform::getFinishedSignal() +{ + assert(outputs.size() == 2); + return outputs.back(); } IProcessor::Status JoiningTransform::prepare() { auto & output = outputs.front(); + auto & on_finish_output = outputs.back(); /// Check can output. if (output.isFinished() || stop_reading) { output.finish(); + on_finish_output.finish(); for (auto & input : inputs) input.close(); return Status::Finished; @@ -93,6 +104,7 @@ IProcessor::Status JoiningTransform::prepare() return Status::Ready; output.finish(); + on_finish_output.finish(); return Status::Finished; } @@ -134,7 +146,7 @@ void JoiningTransform::work() } } - Block block = non_joined_blocks->read(); + Block block = non_joined_blocks->next(); if (!block) { process_non_joined = false; @@ -298,4 +310,132 @@ void FillingRightJoinSideTransform::work() set_totals = for_totals; } + +DelayedJoinedBlocksWorkerTransform::DelayedJoinedBlocksWorkerTransform(Block output_header) + : IProcessor(InputPorts{Block()}, OutputPorts{output_header}) +{ +} + +IProcessor::Status DelayedJoinedBlocksWorkerTransform::prepare() +{ + if (inputs.size() != 1 && outputs.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform must have exactly one input port"); + + auto & output = outputs.front(); + + auto & input = inputs.front(); + + if (output_chunk) + { + input.setNotNeeded(); + + if (!output.canPush()) + return Status::PortFull; + + output.push(std::move(output_chunk)); + output_chunk.clear(); + return Status::PortFull; + } + + if (!task) + { + if (!input.hasData()) + { + input.setNeeded(); + return Status::NeedData; + } + + auto data = input.pullData(true); + if (data.exception) + { + output.pushException(data.exception); + return Status::Finished; + } + + if (!data.chunk.hasChunkInfo()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform must have chunk info"); + task = std::dynamic_pointer_cast(data.chunk.getChunkInfo()); + } + else + { + input.setNotNeeded(); + } + + if (task->finished) + { + input.close(); + output.finish(); + return Status::Finished; + } + + return Status::Ready; +} + +void DelayedJoinedBlocksWorkerTransform::work() +{ + if (!task) + return; + + Block block = task->delayed_blocks->next(); + + if (!block) + { + task.reset(); + return; + } + + // Add block to the output + auto rows = block.rows(); + output_chunk.setColumns(block.getColumns(), rows); +} + +DelayedJoinedBlocksTransform::DelayedJoinedBlocksTransform(size_t num_streams, JoinPtr join_) + : IProcessor(InputPorts{}, OutputPorts(num_streams, Block())) + , join(std::move(join_)) +{ +} + +void DelayedJoinedBlocksTransform::work() +{ + delayed_blocks = join->getDelayedBlocks(); + finished = finished || delayed_blocks == nullptr; +} + + +IProcessor::Status DelayedJoinedBlocksTransform::prepare() +{ + for (auto & output : outputs) + { + if (!output.canPush()) + return Status::PortFull; + } + + if (finished) + { + for (auto & output : outputs) + { + Chunk chunk; + chunk.setChunkInfo(std::make_shared()); + output.push(std::move(chunk)); + output.finish(); + } + + return Status::Finished; + } + + if (delayed_blocks) + { + for (auto & output : outputs) + { + Chunk chunk; + chunk.setChunkInfo(std::make_shared(delayed_blocks)); + output.push(std::move(chunk)); + } + delayed_blocks = nullptr; + return Status::PortFull; + } + + return Status::Ready; +} + } diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h index 0595d035657..e7edff40c56 100644 --- a/src/Processors/Transforms/JoiningTransform.h +++ b/src/Processors/Transforms/JoiningTransform.h @@ -9,6 +9,8 @@ class IJoin; using JoinPtr = std::shared_ptr; class NotJoinedBlocks; +class IBlocksStream; +using IBlocksStreamPtr = std::shared_ptr; /// Join rows to chunk form left table. /// This transform usually has two input ports and one output. @@ -47,10 +49,14 @@ public: bool default_totals_ = false, FinishCounterPtr finish_counter_ = nullptr); + ~JoiningTransform() override; + String getName() const override { return "JoiningTransform"; } static Block transformHeader(Block header, const JoinPtr & join); + OutputPort & getFinishedSignal(); + Status prepare() override; void work() override; @@ -76,7 +82,7 @@ private: ExtraBlockPtr not_processed; FinishCounterPtr finish_counter; - std::shared_ptr non_joined_blocks; + IBlocksStreamPtr non_joined_blocks; size_t max_block_size; Block readExecute(Chunk & chunk); @@ -104,4 +110,55 @@ private: bool set_totals = false; }; + +class DelayedBlocksTask : public ChunkInfo +{ +public: + + explicit DelayedBlocksTask() : finished(true) {} + explicit DelayedBlocksTask(IBlocksStreamPtr delayed_blocks_) : delayed_blocks(std::move(delayed_blocks_)) {} + + IBlocksStreamPtr delayed_blocks = nullptr; + + bool finished = false; +}; + +using DelayedBlocksTaskPtr = std::shared_ptr; + + +/// Reads delayed joined blocks from Join +class DelayedJoinedBlocksTransform : public IProcessor +{ +public: + explicit DelayedJoinedBlocksTransform(size_t num_streams, JoinPtr join_); + + String getName() const override { return "DelayedJoinedBlocksTransform"; } + + Status prepare() override; + void work() override; + +private: + JoinPtr join; + + IBlocksStreamPtr delayed_blocks = nullptr; + bool finished = false; +}; + +class DelayedJoinedBlocksWorkerTransform : public IProcessor +{ +public: + explicit DelayedJoinedBlocksWorkerTransform(Block output_header); + + String getName() const override { return "DelayedJoinedBlocksWorkerTransform"; } + + Status prepare() override; + void work() override; + +private: + DelayedBlocksTaskPtr task; + Chunk output_chunk; + + bool finished = false; +}; + } diff --git a/src/Processors/Transforms/MemoryBoundMerging.h b/src/Processors/Transforms/MemoryBoundMerging.h new file mode 100644 index 00000000000..d4e2cd41e9d --- /dev/null +++ b/src/Processors/Transforms/MemoryBoundMerging.h @@ -0,0 +1,207 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + +/// Has several inputs and single output. +/// Read from inputs merged buckets with aggregated data, sort them by bucket number and block number. +/// Presumption: inputs return chunks with increasing bucket and block number, there is at most one chunk with the given bucket and block number. +class SortingAggregatedForMemoryBoundMergingTransform : public IProcessor +{ +public: + explicit SortingAggregatedForMemoryBoundMergingTransform(const Block & header_, size_t num_inputs_) + : IProcessor(InputPorts(num_inputs_, header_), {header_}) + , header(header_) + , num_inputs(num_inputs_) + , last_chunk_id(num_inputs, {std::numeric_limits::min(), 0}) + , is_input_finished(num_inputs, false) + { + } + + String getName() const override { return "SortingAggregatedForMemoryBoundMergingTransform"; } + + Status prepare() override + { + auto & output = outputs.front(); + + if (output.isFinished()) + { + for (auto & input : inputs) + input.close(); + + return Status::Finished; + } + + if (!output.canPush()) + { + for (auto & input : inputs) + input.setNotNeeded(); + + return Status::PortFull; + } + + /// Push if have chunk that is the next in order + bool pushed_to_output = tryPushChunk(); + + bool need_data = false; + bool all_finished = true; + + /// Try read new chunk + auto in = inputs.begin(); + for (size_t input_num = 0; input_num < num_inputs; ++input_num, ++in) + { + if (in->isFinished()) + { + is_input_finished[input_num] = true; + continue; + } + + /// We want to keep not more than `num_inputs` chunks in memory (and there will be only a single chunk with the given (bucket_id, chunk_num)). + const bool bucket_from_this_input_still_in_memory = chunks.contains(last_chunk_id[input_num]); + if (bucket_from_this_input_still_in_memory) + { + all_finished = false; + continue; + } + + in->setNeeded(); + + if (!in->hasData()) + { + need_data = true; + all_finished = false; + continue; + } + + auto chunk = in->pull(); + addChunk(std::move(chunk), input_num); + + if (in->isFinished()) + { + is_input_finished[input_num] = true; + } + else + { + /// If chunk was pulled, then we need data from this port. + need_data = true; + all_finished = false; + } + } + + if (pushed_to_output) + return Status::PortFull; + + if (tryPushChunk()) + return Status::PortFull; + + if (need_data) + return Status::NeedData; + + if (!all_finished) + throw Exception( + "SortingAggregatedForMemoryBoundMergingTransform has read bucket, but couldn't push it.", ErrorCodes::LOGICAL_ERROR); + + if (overflow_chunk) + { + output.push(std::move(overflow_chunk)); + return Status::PortFull; + } + + output.finish(); + return Status::Finished; + } + +private: + bool tryPushChunk() + { + auto & output = outputs.front(); + + if (chunks.empty()) + return false; + + /// Chunk with min id + auto it = chunks.begin(); + auto current_chunk_id = it->first; + + /// Check if it is actually next in order + for (size_t input = 0; input < num_inputs; ++input) + if (!is_input_finished[input] && last_chunk_id[input] < current_chunk_id) + return false; + + output.push(std::move(it->second)); + chunks.erase(it); + return true; + } + + void addChunk(Chunk chunk, size_t from_input) + { + if (!chunk.hasRows()) + return; + + const auto & info = chunk.getChunkInfo(); + if (!info) + throw Exception( + "Chunk info was not set for chunk in SortingAggregatedForMemoryBoundMergingTransform.", ErrorCodes::LOGICAL_ERROR); + + const auto * agg_info = typeid_cast(info.get()); + if (!agg_info) + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Chunk should have AggregatedChunkInfo in SortingAggregatedForMemoryBoundMergingTransform."); + + Int32 bucket_id = agg_info->bucket_num; + bool is_overflows = agg_info->is_overflows; + UInt64 chunk_num = agg_info->chunk_num; + + if (is_overflows) + overflow_chunk = std::move(chunk); + else + { + const auto chunk_id = ChunkId{bucket_id, chunk_num}; + if (chunks.contains(chunk_id)) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "SortingAggregatedForMemoryBoundMergingTransform already got bucket with number {}", + bucket_id); + } + + chunks[chunk_id] = std::move(chunk); + last_chunk_id[from_input] = chunk_id; + } + } + + struct ChunkId + { + Int32 bucket_id; + UInt64 chunk_num; + + bool operator<(const ChunkId & other) const + { + return std::make_pair(bucket_id, chunk_num) < std::make_pair(other.bucket_id, other.chunk_num); + } + }; + + Block header; + size_t num_inputs; + + std::vector last_chunk_id; + std::vector is_input_finished; + std::map chunks; + Chunk overflow_chunk; +}; + +} diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp index 8471139d9dc..4e90159aa11 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -305,8 +306,9 @@ void GroupingAggregatedTransform::work() } -MergingAggregatedBucketTransform::MergingAggregatedBucketTransform(AggregatingTransformParamsPtr params_) - : ISimpleTransform({}, params_->getHeader(), false), params(std::move(params_)) +MergingAggregatedBucketTransform::MergingAggregatedBucketTransform( + AggregatingTransformParamsPtr params_, const SortDescription & required_sort_description_) + : ISimpleTransform({}, params_->getHeader(), false), params(std::move(params_)), required_sort_description(required_sort_description_) { setInputNotNeededAfterRead(true); } @@ -356,9 +358,14 @@ void MergingAggregatedBucketTransform::transform(Chunk & chunk) auto res_info = std::make_shared(); res_info->is_overflows = chunks_to_merge->is_overflows; res_info->bucket_num = chunks_to_merge->bucket_num; + res_info->chunk_num = chunks_to_merge->chunk_num; chunk.setChunkInfo(std::move(res_info)); auto block = params->aggregator.mergeBlocks(blocks_list, params->final); + + if (!required_sort_description.empty()) + sortBlock(block, required_sort_description); + size_t num_rows = block.rows(); chunk.setColumns(block.getColumns(), num_rows); } diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h index b4a62f8a13e..7c59ad1719f 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h @@ -1,9 +1,10 @@ #pragma once -#include +#include #include +#include #include -#include #include +#include namespace DB @@ -105,7 +106,8 @@ private: class MergingAggregatedBucketTransform : public ISimpleTransform { public: - explicit MergingAggregatedBucketTransform(AggregatingTransformParamsPtr params); + explicit MergingAggregatedBucketTransform( + AggregatingTransformParamsPtr params, const SortDescription & required_sort_description_ = {}); String getName() const override { return "MergingAggregatedBucketTransform"; } protected: @@ -113,6 +115,7 @@ protected: private: AggregatingTransformParamsPtr params; + const SortDescription required_sort_description; }; /// Has several inputs and single output. @@ -142,6 +145,7 @@ struct ChunksToMerge : public ChunkInfo std::unique_ptr chunks; Int32 bucket_num = -1; bool is_overflows = false; + UInt64 chunk_num = 0; // chunk number in order of generation, used during memory bound merging to restore chunks order }; class Pipe; diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index cc484855e76..085399e4941 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -225,13 +225,13 @@ Chain buildPushingToViewsChain( disable_deduplication_for_children = !no_destination && storage->supportsDeduplication(); auto table_id = storage->getStorageID(); - Dependencies dependencies = DatabaseCatalog::instance().getDependencies(table_id); + auto views = DatabaseCatalog::instance().getDependentViews(table_id); /// We need special context for materialized views insertions ContextMutablePtr select_context; ContextMutablePtr insert_context; ViewsDataPtr views_data; - if (!dependencies.empty()) + if (!views.empty()) { select_context = Context::createCopy(context); insert_context = Context::createCopy(context); @@ -253,10 +253,10 @@ Chain buildPushingToViewsChain( std::vector chains; - for (const auto & database_table : dependencies) + for (const auto & view_id : views) { - auto dependent_table = DatabaseCatalog::instance().getTable(database_table, context); - auto dependent_metadata_snapshot = dependent_table->getInMemoryMetadataPtr(); + auto view = DatabaseCatalog::instance().getTable(view_id, context); + auto view_metadata_snapshot = view->getInMemoryMetadataPtr(); ASTPtr query; Chain out; @@ -288,7 +288,7 @@ Chain buildPushingToViewsChain( views_data->thread_status_holder->thread_statuses.push_front(std::move(view_thread_status_ptr)); auto runtime_stats = std::make_unique(); - runtime_stats->target_name = database_table.getFullTableName(); + runtime_stats->target_name = view_id.getFullTableName(); runtime_stats->thread_status = view_thread_status; runtime_stats->event_time = std::chrono::system_clock::now(); runtime_stats->event_status = QueryViewsLogElement::ViewStatus::EXCEPTION_BEFORE_START; @@ -297,7 +297,7 @@ Chain buildPushingToViewsChain( auto & target_name = runtime_stats->target_name; auto * view_counter_ms = &runtime_stats->elapsed_ms; - if (auto * materialized_view = dynamic_cast(dependent_table.get())) + if (auto * materialized_view = dynamic_cast(view.get())) { type = QueryViewsLogElement::ViewType::MATERIALIZED; result_chain.addTableLock(materialized_view->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout)); @@ -305,7 +305,7 @@ Chain buildPushingToViewsChain( StoragePtr inner_table = materialized_view->getTargetTable(); auto inner_table_id = inner_table->getStorageID(); auto inner_metadata_snapshot = inner_table->getInMemoryMetadataPtr(); - query = dependent_metadata_snapshot->getSelectQuery().inner_query; + query = view_metadata_snapshot->getSelectQuery().inner_query; target_name = inner_table_id.getFullTableName(); /// Get list of columns we get from select query. @@ -324,31 +324,31 @@ Chain buildPushingToViewsChain( InterpreterInsertQuery interpreter(nullptr, insert_context, false, false, false); out = interpreter.buildChain(inner_table, inner_metadata_snapshot, insert_columns, thread_status_holder, view_counter_ms); - out.addStorageHolder(dependent_table); + out.addStorageHolder(view); out.addStorageHolder(inner_table); } - else if (auto * live_view = dynamic_cast(dependent_table.get())) + else if (auto * live_view = dynamic_cast(view.get())) { runtime_stats->type = QueryViewsLogElement::ViewType::LIVE; query = live_view->getInnerQuery(); // Used only to log in system.query_views_log out = buildPushingToViewsChain( - dependent_table, dependent_metadata_snapshot, insert_context, ASTPtr(), true, thread_status_holder, view_counter_ms, storage_header); + view, view_metadata_snapshot, insert_context, ASTPtr(), true, thread_status_holder, view_counter_ms, storage_header); } - else if (auto * window_view = dynamic_cast(dependent_table.get())) + else if (auto * window_view = dynamic_cast(view.get())) { runtime_stats->type = QueryViewsLogElement::ViewType::WINDOW; query = window_view->getMergeableQuery(); // Used only to log in system.query_views_log out = buildPushingToViewsChain( - dependent_table, dependent_metadata_snapshot, insert_context, ASTPtr(), true, thread_status_holder, view_counter_ms); + view, view_metadata_snapshot, insert_context, ASTPtr(), true, thread_status_holder, view_counter_ms); } else out = buildPushingToViewsChain( - dependent_table, dependent_metadata_snapshot, insert_context, ASTPtr(), false, thread_status_holder, view_counter_ms); + view, view_metadata_snapshot, insert_context, ASTPtr(), false, thread_status_holder, view_counter_ms); views_data->views.emplace_back(ViewRuntimeData{ //-V614 std::move(query), out.getInputHeader(), - database_table, + view_id, nullptr, std::move(runtime_stats)}); @@ -367,7 +367,7 @@ Chain buildPushingToViewsChain( if (!no_destination) { context->getQueryContext()->addQueryAccessInfo( - backQuoteIfNeed(database_table.getDatabaseName()), views_data->views.back().runtime_stats->target_name, {}, "", database_table.getFullTableName()); + backQuoteIfNeed(view_id.getDatabaseName()), views_data->views.back().runtime_stats->target_name, {}, "", view_id.getFullTableName()); } } diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index 812bd155b42..626296834a2 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -22,7 +22,8 @@ #include #include #include -#include "Core/SortDescription.h" +#include +#include #include #include #include @@ -383,7 +384,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe /// Collect the NEW processors for the right pipeline. QueryPipelineProcessorsCollector collector(*right); /// Remember the last step of the right pipeline. - ExpressionStep* step = typeid_cast(right->pipe.processors->back()->getQueryPlanStep()); + ExpressionStep * step = typeid_cast(right->pipe.processors->back()->getQueryPlanStep()); if (!step) { throw Exception(ErrorCodes::LOGICAL_ERROR, "The top step of the right pipeline should be ExpressionStep"); @@ -391,6 +392,10 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe /// In case joined subquery has totals, and we don't, add default chunk to totals. bool default_totals = false; + + if (!join->supportTotals() && (left->hasTotals() || right->hasTotals())) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Current join algorithm is supported only for pipelines without totals"); + if (!left->hasTotals() && right->hasTotals()) { left->addDefaultTotals(); @@ -453,26 +458,94 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe auto lit = left->pipe.output_ports.begin(); auto rit = right->pipe.output_ports.begin(); + + std::vector joined_output_ports; + std::vector delayed_root_output_ports; + + std::shared_ptr delayed_root = nullptr; + if (join->hasDelayedBlocks()) + { + delayed_root = std::make_shared(num_streams, join); + if (!delayed_root->getInputs().empty() || delayed_root->getOutputs().size() != num_streams) + throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksTransform should have no inputs and {} outputs, but has {} inputs and {} outputs", + num_streams, delayed_root->getInputs().size(), delayed_root->getOutputs().size()); + + if (collected_processors) + collected_processors->emplace_back(delayed_root); + left->pipe.processors->emplace_back(delayed_root); + + for (auto & outport : delayed_root->getOutputs()) + delayed_root_output_ports.emplace_back(&outport); + } + + + Block left_header = left->getHeader(); + Block joined_header = JoiningTransform::transformHeader(left_header, join); + for (size_t i = 0; i < num_streams; ++i) { auto joining = std::make_shared( - left->getHeader(), output_header, join, max_block_size, false, default_totals, finish_counter); + left_header, output_header, join, max_block_size, false, default_totals, finish_counter); + connect(**lit, joining->getInputs().front()); connect(**rit, joining->getInputs().back()); - *lit = &joining->getOutputs().front(); + if (delayed_root) + { + // Process delayed joined blocks when all JoiningTransform are finished. + auto delayed = std::make_shared(joined_header); + if (delayed->getInputs().size() != 1 || delayed->getOutputs().size() != 1) + throw Exception("DelayedJoinedBlocksWorkerTransform should have one input and one output", ErrorCodes::LOGICAL_ERROR); + + connect(*delayed_root_output_ports[i], delayed->getInputs().front()); + + joined_output_ports.push_back(&joining->getOutputs().front()); + joined_output_ports.push_back(&delayed->getOutputs().front()); + + if (collected_processors) + collected_processors->emplace_back(delayed); + left->pipe.processors->emplace_back(std::move(delayed)); + } + else + { + *lit = &joining->getOutputs().front(); + } + ++lit; ++rit; - if (collected_processors) collected_processors->emplace_back(joining); left->pipe.processors->emplace_back(std::move(joining)); } + if (delayed_root) + { + // Process DelayedJoinedBlocksTransform after all JoiningTransforms. + DelayedPortsProcessor::PortNumbers delayed_ports_numbers; + delayed_ports_numbers.reserve(joined_output_ports.size() / 2); + for (size_t i = 1; i < joined_output_ports.size(); i += 2) + delayed_ports_numbers.push_back(i); + + auto delayed_processor = std::make_shared(joined_header, 2 * num_streams, delayed_ports_numbers); + if (collected_processors) + collected_processors->emplace_back(delayed_processor); + left->pipe.processors->emplace_back(delayed_processor); + + // Connect @delayed_processor ports with inputs (JoiningTransforms & DelayedJoinedBlocksTransforms) / pipe outputs + auto next_delayed_input = delayed_processor->getInputs().begin(); + for (OutputPort * port : joined_output_ports) + connect(*port, *next_delayed_input++); + left->pipe.output_ports.clear(); + for (OutputPort & port : delayed_processor->getOutputs()) + left->pipe.output_ports.push_back(&port); + left->pipe.header = joined_header; + left->resize(num_streams); + } + if (left->hasTotals()) { - auto joining = std::make_shared(left->getHeader(), output_header, join, max_block_size, true, default_totals); + auto joining = std::make_shared(left_header, output_header, join, max_block_size, true, default_totals); connect(*left->pipe.totals_port, joining->getInputs().front()); connect(**rit, joining->getInputs().back()); left->pipe.totals_port = &joining->getOutputs().front(); diff --git a/src/Server/HTTP/HTTPContext.h b/src/Server/HTTP/HTTPContext.h new file mode 100644 index 00000000000..09c46ed188c --- /dev/null +++ b/src/Server/HTTP/HTTPContext.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace DB +{ + +struct IHTTPContext +{ + virtual uint64_t getMaxHstsAge() const = 0; + virtual uint64_t getMaxUriSize() const = 0; + virtual uint64_t getMaxFields() const = 0; + virtual uint64_t getMaxFieldNameSize() const = 0; + virtual uint64_t getMaxFieldValueSize() const = 0; + virtual uint64_t getMaxChunkSize() const = 0; + virtual Poco::Timespan getReceiveTimeout() const = 0; + virtual Poco::Timespan getSendTimeout() const = 0; + + virtual ~IHTTPContext() = default; +}; + +using HTTPContextPtr = std::shared_ptr; + +} diff --git a/src/Server/HTTP/HTTPServer.cpp b/src/Server/HTTP/HTTPServer.cpp index 2e91fad1c0f..46734933263 100644 --- a/src/Server/HTTP/HTTPServer.cpp +++ b/src/Server/HTTP/HTTPServer.cpp @@ -6,7 +6,7 @@ namespace DB { HTTPServer::HTTPServer( - ContextPtr context, + HTTPContextPtr context, HTTPRequestHandlerFactoryPtr factory_, Poco::ThreadPool & thread_pool, Poco::Net::ServerSocket & socket_, diff --git a/src/Server/HTTP/HTTPServer.h b/src/Server/HTTP/HTTPServer.h index 07ad54d267f..adfb21e7c62 100644 --- a/src/Server/HTTP/HTTPServer.h +++ b/src/Server/HTTP/HTTPServer.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -11,13 +12,11 @@ namespace DB { -class Context; - class HTTPServer : public TCPServer { public: explicit HTTPServer( - ContextPtr context, + HTTPContextPtr context, HTTPRequestHandlerFactoryPtr factory, Poco::ThreadPool & thread_pool, Poco::Net::ServerSocket & socket, diff --git a/src/Server/HTTP/HTTPServerConnection.cpp b/src/Server/HTTP/HTTPServerConnection.cpp index 92a994b3a4e..926d37a11ee 100644 --- a/src/Server/HTTP/HTTPServerConnection.cpp +++ b/src/Server/HTTP/HTTPServerConnection.cpp @@ -7,12 +7,12 @@ namespace DB { HTTPServerConnection::HTTPServerConnection( - ContextPtr context_, + HTTPContextPtr context_, TCPServer & tcp_server_, const Poco::Net::StreamSocket & socket, Poco::Net::HTTPServerParams::Ptr params_, HTTPRequestHandlerFactoryPtr factory_) - : TCPServerConnection(socket), context(Context::createCopy(context_)), tcp_server(tcp_server_), params(params_), factory(factory_), stopped(false) + : TCPServerConnection(socket), context(std::move(context_)), tcp_server(tcp_server_), params(params_), factory(factory_), stopped(false) { poco_check_ptr(factory); } @@ -36,7 +36,7 @@ void HTTPServerConnection::run() if (request.isSecure()) { - size_t hsts_max_age = context->getSettingsRef().hsts_max_age.value; + size_t hsts_max_age = context->getMaxHstsAge(); if (hsts_max_age > 0) response.add("Strict-Transport-Security", "max-age=" + std::to_string(hsts_max_age)); diff --git a/src/Server/HTTP/HTTPServerConnection.h b/src/Server/HTTP/HTTPServerConnection.h index db3969f6ffb..cce4f44f203 100644 --- a/src/Server/HTTP/HTTPServerConnection.h +++ b/src/Server/HTTP/HTTPServerConnection.h @@ -1,7 +1,7 @@ #pragma once -#include #include +#include #include #include @@ -15,7 +15,7 @@ class HTTPServerConnection : public Poco::Net::TCPServerConnection { public: HTTPServerConnection( - ContextPtr context, + HTTPContextPtr context, TCPServer & tcp_server, const Poco::Net::StreamSocket & socket, Poco::Net::HTTPServerParams::Ptr params, @@ -27,7 +27,7 @@ protected: static void sendErrorResponse(Poco::Net::HTTPServerSession & session, Poco::Net::HTTPResponse::HTTPStatus status); private: - ContextPtr context; + HTTPContextPtr context; TCPServer & tcp_server; Poco::Net::HTTPServerParams::Ptr params; HTTPRequestHandlerFactoryPtr factory; diff --git a/src/Server/HTTP/HTTPServerConnectionFactory.cpp b/src/Server/HTTP/HTTPServerConnectionFactory.cpp index 008da222c79..7e4edbbf542 100644 --- a/src/Server/HTTP/HTTPServerConnectionFactory.cpp +++ b/src/Server/HTTP/HTTPServerConnectionFactory.cpp @@ -5,8 +5,8 @@ namespace DB { HTTPServerConnectionFactory::HTTPServerConnectionFactory( - ContextPtr context_, Poco::Net::HTTPServerParams::Ptr params_, HTTPRequestHandlerFactoryPtr factory_) - : context(Context::createCopy(context_)), params(params_), factory(factory_) + HTTPContextPtr context_, Poco::Net::HTTPServerParams::Ptr params_, HTTPRequestHandlerFactoryPtr factory_) + : context(std::move(context_)), params(params_), factory(factory_) { poco_check_ptr(factory); } diff --git a/src/Server/HTTP/HTTPServerConnectionFactory.h b/src/Server/HTTP/HTTPServerConnectionFactory.h index a19dc6d4d5c..03648ce7be7 100644 --- a/src/Server/HTTP/HTTPServerConnectionFactory.h +++ b/src/Server/HTTP/HTTPServerConnectionFactory.h @@ -1,7 +1,7 @@ #pragma once -#include #include +#include #include #include @@ -12,12 +12,12 @@ namespace DB class HTTPServerConnectionFactory : public TCPServerConnectionFactory { public: - HTTPServerConnectionFactory(ContextPtr context, Poco::Net::HTTPServerParams::Ptr params, HTTPRequestHandlerFactoryPtr factory); + HTTPServerConnectionFactory(HTTPContextPtr context, Poco::Net::HTTPServerParams::Ptr params, HTTPRequestHandlerFactoryPtr factory); Poco::Net::TCPServerConnection * createConnection(const Poco::Net::StreamSocket & socket, TCPServer & tcp_server) override; private: - ContextPtr context; + HTTPContextPtr context; Poco::Net::HTTPServerParams::Ptr params; HTTPRequestHandlerFactoryPtr factory; }; diff --git a/src/Server/HTTP/HTTPServerRequest.cpp b/src/Server/HTTP/HTTPServerRequest.cpp index 3b8df07b772..a82eb95aee1 100644 --- a/src/Server/HTTP/HTTPServerRequest.cpp +++ b/src/Server/HTTP/HTTPServerRequest.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -21,11 +20,11 @@ namespace DB { -HTTPServerRequest::HTTPServerRequest(ContextPtr context, HTTPServerResponse & response, Poco::Net::HTTPServerSession & session) - : max_uri_size(context->getSettingsRef().http_max_uri_size) - , max_fields_number(context->getSettingsRef().http_max_fields) - , max_field_name_size(context->getSettingsRef().http_max_field_name_size) - , max_field_value_size(context->getSettingsRef().http_max_field_value_size) +HTTPServerRequest::HTTPServerRequest(HTTPContextPtr context, HTTPServerResponse & response, Poco::Net::HTTPServerSession & session) + : max_uri_size(context->getMaxUriSize()) + , max_fields_number(context->getMaxFields()) + , max_field_name_size(context->getMaxFieldNameSize()) + , max_field_value_size(context->getMaxFieldValueSize()) { response.attachRequest(this); @@ -34,8 +33,8 @@ HTTPServerRequest::HTTPServerRequest(ContextPtr context, HTTPServerResponse & re server_address = session.serverAddress(); secure = session.socket().secure(); - auto receive_timeout = context->getSettingsRef().http_receive_timeout; - auto send_timeout = context->getSettingsRef().http_send_timeout; + auto receive_timeout = context->getReceiveTimeout(); + auto send_timeout = context->getSendTimeout(); session.socket().setReceiveTimeout(receive_timeout); session.socket().setSendTimeout(send_timeout); @@ -46,7 +45,7 @@ HTTPServerRequest::HTTPServerRequest(ContextPtr context, HTTPServerResponse & re readRequest(*in); /// Try parse according to RFC7230 if (getChunkedTransferEncoding()) - stream = std::make_unique(std::move(in), context->getSettingsRef().http_max_chunk_size); + stream = std::make_unique(std::move(in), context->getMaxChunkSize()); else if (hasContentLength()) stream = std::make_unique(std::move(in), getContentLength(), false); else if (getMethod() != HTTPRequest::HTTP_GET && getMethod() != HTTPRequest::HTTP_HEAD && getMethod() != HTTPRequest::HTTP_DELETE) diff --git a/src/Server/HTTP/HTTPServerRequest.h b/src/Server/HTTP/HTTPServerRequest.h index 7ddbd296280..1f38334c745 100644 --- a/src/Server/HTTP/HTTPServerRequest.h +++ b/src/Server/HTTP/HTTPServerRequest.h @@ -3,6 +3,7 @@ #include #include #include +#include #include "config.h" #include @@ -18,7 +19,7 @@ class ReadBufferFromPocoSocket; class HTTPServerRequest : public HTTPRequest { public: - HTTPServerRequest(ContextPtr context, HTTPServerResponse & response, Poco::Net::HTTPServerSession & session); + HTTPServerRequest(HTTPContextPtr context, HTTPServerResponse & response, Poco::Net::HTTPServerSession & session); /// FIXME: it's a little bit inconvenient interface. The rationale is that all other ReadBuffer's wrap each other /// via unique_ptr - but we can't inherit HTTPServerRequest from ReadBuffer and pass it around, diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp index ac8f8332a9e..e4da7941b50 100644 --- a/src/Server/HTTPHandlerFactory.cpp +++ b/src/Server/HTTPHandlerFactory.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include @@ -33,35 +32,6 @@ static void addDefaultHandlersFactory( const Poco::Util::AbstractConfiguration & config, AsynchronousMetrics & async_metrics); -HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(const std::string & name_) - : log(&Poco::Logger::get(name_)), name(name_) -{ -} - -std::unique_ptr HTTPRequestHandlerFactoryMain::createRequestHandler(const HTTPServerRequest & request) -{ - LOG_TRACE(log, "HTTP Request for {}. Method: {}, Address: {}, User-Agent: {}{}, Content Type: {}, Transfer Encoding: {}, X-Forwarded-For: {}", - name, request.getMethod(), request.clientAddress().toString(), request.get("User-Agent", "(none)"), - (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")), - request.getContentType(), request.getTransferEncoding(), request.get("X-Forwarded-For", "(none)")); - - for (auto & handler_factory : child_factories) - { - auto handler = handler_factory->createRequestHandler(request); - if (handler) - return handler; - } - - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET - || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD - || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) - { - return std::unique_ptr(new NotFoundHandler); - } - - return nullptr; -} - static inline auto createHandlersFactoryFromConfig( IServer & server, const Poco::Util::AbstractConfiguration & config, @@ -144,15 +114,7 @@ HTTPRequestHandlerFactoryPtr createHandlerFactory(IServer & server, const Poco:: else if (name == "InterserverIOHTTPHandler-factory" || name == "InterserverIOHTTPSHandler-factory") return createInterserverHTTPHandlerFactory(server, name); else if (name == "PrometheusHandler-factory") - { - auto factory = std::make_shared(name); - auto handler = std::make_shared>( - server, PrometheusMetricsWriter(config, "prometheus", async_metrics)); - handler->attachStrictPath(config.getString("prometheus.endpoint", "/metrics")); - handler->allowGetAndHeadRequest(); - factory->addHandler(handler); - return factory; - } + return createPrometheusMainHandlerFactory(server, config, async_metrics, name); throw Exception("LOGICAL ERROR: Unknown HTTP handler factory name.", ErrorCodes::LOGICAL_ERROR); } diff --git a/src/Server/HTTPHandlerFactory.h b/src/Server/HTTPHandlerFactory.h index 9f306e787b0..f56c712c615 100644 --- a/src/Server/HTTPHandlerFactory.h +++ b/src/Server/HTTPHandlerFactory.h @@ -1,9 +1,10 @@ #pragma once -#include +#include #include #include #include +#include #include #include @@ -19,23 +20,6 @@ namespace ErrorCodes class IServer; -/// Handle request using child handlers -class HTTPRequestHandlerFactoryMain : public HTTPRequestHandlerFactory -{ -public: - explicit HTTPRequestHandlerFactoryMain(const std::string & name_); - - void addHandler(HTTPRequestHandlerFactoryPtr child_factory) { child_factories.emplace_back(child_factory); } - - std::unique_ptr createRequestHandler(const HTTPServerRequest & request) override; - -private: - Poco::Logger * log; - std::string name; - - std::vector child_factories; -}; - template class HandlingRuleHTTPHandlerFactory : public HTTPRequestHandlerFactory { @@ -148,6 +132,12 @@ createPrometheusHandlerFactory(IServer & server, AsynchronousMetrics & async_metrics, const std::string & config_prefix); +HTTPRequestHandlerFactoryPtr +createPrometheusMainHandlerFactory(IServer & server, + const Poco::Util::AbstractConfiguration & config, + AsynchronousMetrics & async_metrics, + const std::string & name); + /// @param server - used in handlers to check IServer::isCancelled() /// @param config - not the same as server.config(), since it can be newer /// @param async_metrics - used for prometheus (in case of prometheus.asynchronous_metrics=true) diff --git a/src/Server/HTTPRequestHandlerFactoryMain.cpp b/src/Server/HTTPRequestHandlerFactoryMain.cpp new file mode 100644 index 00000000000..61a2909d30f --- /dev/null +++ b/src/Server/HTTPRequestHandlerFactoryMain.cpp @@ -0,0 +1,38 @@ +#include +#include + +#include + +namespace DB +{ + +HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(const std::string & name_) + : log(&Poco::Logger::get(name_)), name(name_) +{ +} + +std::unique_ptr HTTPRequestHandlerFactoryMain::createRequestHandler(const HTTPServerRequest & request) +{ + LOG_TRACE(log, "HTTP Request for {}. Method: {}, Address: {}, User-Agent: {}{}, Content Type: {}, Transfer Encoding: {}, X-Forwarded-For: {}", + name, request.getMethod(), request.clientAddress().toString(), request.get("User-Agent", "(none)"), + (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")), + request.getContentType(), request.getTransferEncoding(), request.get("X-Forwarded-For", "(none)")); + + for (auto & handler_factory : child_factories) + { + auto handler = handler_factory->createRequestHandler(request); + if (handler) + return handler; + } + + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET + || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD + || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) + { + return std::unique_ptr(new NotFoundHandler); + } + + return nullptr; +} + +} diff --git a/src/Server/HTTPRequestHandlerFactoryMain.h b/src/Server/HTTPRequestHandlerFactoryMain.h new file mode 100644 index 00000000000..b0e57bd6b3b --- /dev/null +++ b/src/Server/HTTPRequestHandlerFactoryMain.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +#include + +namespace DB +{ + +/// Handle request using child handlers +class HTTPRequestHandlerFactoryMain : public HTTPRequestHandlerFactory +{ +public: + explicit HTTPRequestHandlerFactoryMain(const std::string & name_); + + void addHandler(HTTPRequestHandlerFactoryPtr child_factory) { child_factories.emplace_back(child_factory); } + + std::unique_ptr createRequestHandler(const HTTPServerRequest & request) override; + +private: + Poco::Logger * log; + std::string name; + + std::vector child_factories; +}; + +} diff --git a/src/Server/PrometheusMetricsWriter.h b/src/Server/PrometheusMetricsWriter.h index 0c2dde1f66f..b4f6ab57def 100644 --- a/src/Server/PrometheusMetricsWriter.h +++ b/src/Server/PrometheusMetricsWriter.h @@ -2,7 +2,7 @@ #include -#include +#include #include #include diff --git a/src/Server/PrometheusRequestHandler.cpp b/src/Server/PrometheusRequestHandler.cpp index 896efcca674..79025624206 100644 --- a/src/Server/PrometheusRequestHandler.cpp +++ b/src/Server/PrometheusRequestHandler.cpp @@ -53,4 +53,19 @@ createPrometheusHandlerFactory(IServer & server, return factory; } +HTTPRequestHandlerFactoryPtr +createPrometheusMainHandlerFactory(IServer & server, + const Poco::Util::AbstractConfiguration & config, + AsynchronousMetrics & async_metrics, + const std::string & name) +{ + auto factory = std::make_shared(name); + auto handler = std::make_shared>( + server, PrometheusMetricsWriter(config, "prometheus", async_metrics)); + handler->attachStrictPath(config.getString("prometheus.endpoint", "/metrics")); + handler->allowGetAndHeadRequest(); + factory->addHandler(handler); + return factory; +} + } diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index a91a34e6a95..1793003c6fb 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -30,7 +30,6 @@ #include #include - namespace DB { @@ -1024,6 +1023,7 @@ void AlterCommands::prepare(const StorageInMemoryMetadata & metadata) command.ignore = true; } } + prepared = true; } diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index f1300dfd940..39e91e19014 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -572,7 +572,6 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri std::map StorageDistributedDirectoryMonitor::getFiles() { std::map files; - size_t new_bytes_count = 0; fs::directory_iterator end; for (fs::directory_iterator it{path}; it != end; ++it) @@ -581,23 +580,9 @@ std::map StorageDistributedDirectoryMonitor::getFiles() if (!it->is_directory() && startsWith(fs::path(file_path_str).extension(), ".bin")) { files[parse(fs::path(file_path_str).stem())] = file_path_str; - new_bytes_count += fs::file_size(fs::path(file_path_str)); } } - { - std::lock_guard status_lock(status_mutex); - - if (status.files_count != files.size()) - LOG_TRACE(log, "Files set to {} (was {})", files.size(), status.files_count); - if (status.bytes_count != new_bytes_count) - LOG_TRACE(log, "Bytes set to {} (was {})", new_bytes_count, status.bytes_count); - - metric_pending_files.changeTo(files.size()); - status.files_count = files.size(); - status.bytes_count = new_bytes_count; - } - return files; } bool StorageDistributedDirectoryMonitor::processFiles(const std::map & files) diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 8099a7f2002..38ff06f4744 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include @@ -290,6 +291,10 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si auto thread_group = CurrentThread::getGroup(); return [this, thread_group, &job, ¤t_block, num_shards]() { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); OpenTelemetry::SpanHolder span(__PRETTY_FUNCTION__); if (thread_group) diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 722843a7ab6..0f4563b6f35 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -547,23 +547,23 @@ size_t StorageFileLog::getPollTimeoutMillisecond() const bool StorageFileLog::checkDependencies(const StorageID & table_id) { // Check if all dependencies are attached - auto dependencies = DatabaseCatalog::instance().getDependencies(table_id); - if (dependencies.empty()) + auto view_ids = DatabaseCatalog::instance().getDependentViews(table_id); + if (view_ids.empty()) return true; - for (const auto & storage : dependencies) + for (const auto & view_id : view_ids) { - auto table = DatabaseCatalog::instance().tryGetTable(storage, getContext()); - if (!table) + auto view = DatabaseCatalog::instance().tryGetTable(view_id, getContext()); + if (!view) return false; // If it materialized view, check it's target table - auto * materialized_view = dynamic_cast(table.get()); + auto * materialized_view = dynamic_cast(view.get()); if (materialized_view && !materialized_view->tryGetTargetTable()) return false; // Check all its dependencies - if (!checkDependencies(storage)) + if (!checkDependencies(view_id)) return false; } @@ -574,7 +574,7 @@ size_t StorageFileLog::getTableDependentCount() const { auto table_id = getStorageID(); // Check if at least one direct dependency is attached - return DatabaseCatalog::instance().getDependencies(table_id).size(); + return DatabaseCatalog::instance().getDependentViews(table_id).size(); } void StorageFileLog::threadFunc() diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 47d7382f7ca..3fb7be5b697 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -773,7 +773,6 @@ Pipe StorageHive::read( sources_info->partition_name_types = partition_name_types; const auto header_block = storage_snapshot->metadata->getSampleBlock(); - bool support_subset_columns = supportsSubcolumns(); auto settings = context_->getSettingsRef(); auto case_insensitive_matching = [&]() -> bool @@ -793,15 +792,14 @@ Pipe StorageHive::read( sample_block.insert(header_block.getByName(column)); continue; } - else if (support_subset_columns) + + auto subset_column = nested_columns_extractor.extractColumn(column); + if (subset_column) { - auto subset_column = nested_columns_extractor.extractColumn(column); - if (subset_column) - { - sample_block.insert(std::move(*subset_column)); - continue; - } + sample_block.insert(std::move(*subset_column)); + continue; } + if (column == "_path") sources_info->need_path_column = true; if (column == "_file") diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index bc2d38de215..76100624d51 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -232,16 +232,16 @@ Names IStorage::getAllRegisteredNames() const NameDependencies IStorage::getDependentViewsByColumn(ContextPtr context) const { NameDependencies name_deps; - auto dependencies = DatabaseCatalog::instance().getDependencies(storage_id); - for (const auto & depend_id : dependencies) + auto view_ids = DatabaseCatalog::instance().getDependentViews(storage_id); + for (const auto & view_id : view_ids) { - auto depend_table = DatabaseCatalog::instance().getTable(depend_id, context); - if (depend_table->getInMemoryMetadataPtr()->select.inner_query) + auto view = DatabaseCatalog::instance().getTable(view_id, context); + if (view->getInMemoryMetadataPtr()->select.inner_query) { - const auto & select_query = depend_table->getInMemoryMetadataPtr()->select.inner_query; + const auto & select_query = view->getInMemoryMetadataPtr()->select.inner_query; auto required_columns = InterpreterSelectQuery(select_query, context, SelectQueryOptions{}.noModify()).getRequiredColumns(); for (const auto & col_name : required_columns) - name_deps[col_name].push_back(depend_id.table_name); + name_deps[col_name].push_back(view_id.table_name); } } return name_deps; diff --git a/src/Storages/KVStorageUtils.cpp b/src/Storages/KVStorageUtils.cpp index 41aa91eef31..7ec1340e339 100644 --- a/src/Storages/KVStorageUtils.cpp +++ b/src/Storages/KVStorageUtils.cpp @@ -140,7 +140,7 @@ std::vector serializeKeysToRawString( { std::string & serialized_key = result.emplace_back(); WriteBufferFromString wb(serialized_key); - key_column_type->getDefaultSerialization()->serializeBinary(*it, wb); + key_column_type->getDefaultSerialization()->serializeBinary(*it, wb, {}); wb.finalize(); ++it; @@ -165,7 +165,7 @@ std::vector serializeKeysToRawString(const ColumnWithTypeAndName & Field field; keys.column->get(i, field); /// TODO(@vdimir): use serializeBinaryBulk - keys.type->getDefaultSerialization()->serializeBinary(field, wb); + keys.type->getDefaultSerialization()->serializeBinary(field, wb, {}); wb.finalize(); } return result; diff --git a/src/Storages/KVStorageUtils.h b/src/Storages/KVStorageUtils.h index e3216164869..0574539f4c7 100644 --- a/src/Storages/KVStorageUtils.h +++ b/src/Storages/KVStorageUtils.h @@ -30,7 +30,7 @@ void fillColumns(const K & key, const V & value, size_t key_pos, const Block & h for (size_t i = 0; i < header.columns(); ++i) { const auto & serialization = header.getByPosition(i).type->getDefaultSerialization(); - serialization->deserializeBinary(*columns[i], i == key_pos ? key_buffer : value_buffer); + serialization->deserializeBinary(*columns[i], i == key_pos ? key_buffer : value_buffer, {}); } } diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 8e4dd78379e..77afa7ba623 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -584,24 +584,24 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) bool StorageKafka::checkDependencies(const StorageID & table_id) { // Check if all dependencies are attached - auto dependencies = DatabaseCatalog::instance().getDependencies(table_id); - if (dependencies.empty()) + auto view_ids = DatabaseCatalog::instance().getDependentViews(table_id); + if (view_ids.empty()) return true; // Check the dependencies are ready? - for (const auto & db_tab : dependencies) + for (const auto & view_id : view_ids) { - auto table = DatabaseCatalog::instance().tryGetTable(db_tab, getContext()); - if (!table) + auto view = DatabaseCatalog::instance().tryGetTable(view_id, getContext()); + if (!view) return false; // If it materialized view, check it's target table - auto * materialized_view = dynamic_cast(table.get()); + auto * materialized_view = dynamic_cast(view.get()); if (materialized_view && !materialized_view->tryGetTargetTable()) return false; // Check all its dependencies - if (!checkDependencies(db_tab)) + if (!checkDependencies(view_id)) return false; } @@ -616,8 +616,8 @@ void StorageKafka::threadFunc(size_t idx) { auto table_id = getStorageID(); // Check if at least one direct dependency is attached - size_t dependencies_count = DatabaseCatalog::instance().getDependencies(table_id).size(); - if (dependencies_count) + size_t num_views = DatabaseCatalog::instance().getDependentViews(table_id).size(); + if (num_views) { auto start_time = std::chrono::steady_clock::now(); @@ -629,7 +629,7 @@ void StorageKafka::threadFunc(size_t idx) if (!checkDependencies(table_id)) break; - LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count); + LOG_DEBUG(log, "Started streaming to {} attached views", num_views); // Exit the loop & reschedule if some stream stalled auto some_stream_is_stalled = streamToViews(); diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 3d27205d638..c92968e4bcc 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -304,7 +304,7 @@ StorageLiveView::StorageLiveView( auto inner_query_tmp = inner_query->clone(); select_table_id = extractDependentTable(inner_query_tmp, getContext(), table_id_.table_name, inner_subquery); - DatabaseCatalog::instance().addDependency(select_table_id, table_id_); + DatabaseCatalog::instance().addViewDependency(select_table_id, table_id_); if (query.live_view_periodic_refresh) { @@ -434,11 +434,11 @@ bool StorageLiveView::getNewBlocks() void StorageLiveView::checkTableCanBeDropped() const { auto table_id = getStorageID(); - Dependencies dependencies = DatabaseCatalog::instance().getDependencies(table_id); - if (!dependencies.empty()) + auto view_ids = DatabaseCatalog::instance().getDependentViews(table_id); + if (!view_ids.empty()) { - StorageID dependent_table_id = dependencies.front(); - throw Exception("Table has dependency " + dependent_table_id.getNameForLogs(), ErrorCodes::TABLE_WAS_NOT_DROPPED); + StorageID view_id = *view_ids.begin(); + throw Exception(ErrorCodes::TABLE_WAS_NOT_DROPPED, "Table has dependency {}", view_id); } } @@ -455,7 +455,7 @@ void StorageLiveView::shutdown() if (is_periodically_refreshed) periodic_refresh_task->deactivate(); - DatabaseCatalog::instance().removeDependency(select_table_id, getStorageID()); + DatabaseCatalog::instance().removeViewDependency(select_table_id, getStorageID()); } StorageLiveView::~StorageLiveView() @@ -466,7 +466,7 @@ StorageLiveView::~StorageLiveView() void StorageLiveView::drop() { auto table_id = getStorageID(); - DatabaseCatalog::instance().removeDependency(select_table_id, table_id); + DatabaseCatalog::instance().removeViewDependency(select_table_id, table_id); std::lock_guard lock(mutex); is_dropped = true; diff --git a/src/Storages/MergeTree/ActiveDataPartSet.cpp b/src/Storages/MergeTree/ActiveDataPartSet.cpp index 7d6b75557ed..a482dd21099 100644 --- a/src/Storages/MergeTree/ActiveDataPartSet.cpp +++ b/src/Storages/MergeTree/ActiveDataPartSet.cpp @@ -65,7 +65,7 @@ bool ActiveDataPartSet::add(const String & name, Strings * out_replaced_parts) } if (it != part_info_to_name.end() && !part_info.isDisjoint(it->first)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} intersects next part {}. It is a bug or a result of manual intervention in the ZooKeeper data.", name, it->first.getPartName()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} intersects part {}. It is a bug or a result of manual intervention in the ZooKeeper data.", name, it->first.getPartName()); part_info_to_name.emplace(part_info, name); return true; diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 7b36a9873e4..347ea16950e 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -101,6 +101,7 @@ public: bool isValid() const override { return it->isValid(); } bool isFile() const override { return isValid() && disk->isFile(it->path()); } std::string name() const override { return it->name(); } + std::string path() const override { return it->path(); } private: DiskPtr disk; @@ -259,9 +260,17 @@ void DataPartStorageOnDisk::remove( std::string proj_dir_name = projection.name + proj_suffix; projection_directories.emplace(proj_dir_name); + NameSet files_not_to_remove_for_projection; + for (const auto & file_name : can_remove_description->files_not_to_remove) + { + if (file_name.starts_with(proj_dir_name)) + files_not_to_remove_for_projection.emplace(fs::path(file_name).filename()); + } + LOG_DEBUG(log, "Will not remove files [{}] for projection {}", fmt::join(files_not_to_remove_for_projection, ", "), projection.name); + clearDirectory( fs::path(to) / proj_dir_name, - can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove, projection.checksums, {}, is_temp, state, log, true); + can_remove_description->can_remove_anything, files_not_to_remove_for_projection, projection.checksums, {}, is_temp, state, log, true); } /// It is possible that we are removing the part which have a written but not loaded projection. @@ -372,7 +381,12 @@ std::optional DataPartStorageOnDisk::getRelativePathForPrefix(Poco::Logg for (int try_no = 0; try_no < 10; ++try_no) { - res = (prefix.empty() ? "" : prefix + "_") + part_dir + (try_no ? "_try" + DB::toString(try_no) : ""); + if (prefix.empty()) + res = part_dir + (try_no ? "_try" + DB::toString(try_no) : ""); + else if (prefix.ends_with("_")) + res = prefix + part_dir + (try_no ? "_try" + DB::toString(try_no) : ""); + else + res = prefix + "_" + part_dir + (try_no ? "_try" + DB::toString(try_no) : ""); if (!volume->getDisk()->exists(full_relative_path / res)) return res; diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index c6669908db4..53ee2738fc6 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -39,6 +39,9 @@ public: /// Name of the file that the iterator currently points to. virtual std::string name() const = 0; + /// Path of the file that the iterator currently points to. + virtual std::string path() const = 0; + virtual ~IDataPartStorageIterator() = default; }; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index f38105ce1f6..660b1baca06 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -86,9 +86,9 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Par auto serialization = minmax_column_types[i]->getDefaultSerialization(); Field min_val; - serialization->deserializeBinary(min_val, *file); + serialization->deserializeBinary(min_val, *file, {}); Field max_val; - serialization->deserializeBinary(max_val, *file); + serialization->deserializeBinary(max_val, *file, {}); // NULL_LAST if (min_val.isNull()) @@ -134,8 +134,8 @@ IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::s auto out = part_storage.writeFile(file_name, DBMS_DEFAULT_BUFFER_SIZE, {}); HashingWriteBuffer out_hashing(*out); - serialization->serializeBinary(hyperrectangle[i].left, out_hashing); - serialization->serializeBinary(hyperrectangle[i].right, out_hashing); + serialization->serializeBinary(hyperrectangle[i].left, out_hashing, {}); + serialization->serializeBinary(hyperrectangle[i].right, out_hashing, {}); out_hashing.next(); out_checksums.files[file_name].file_size = out_hashing.count(); out_checksums.files[file_name].file_hash = out_hashing.getHash(); @@ -755,7 +755,7 @@ void IMergeTreeDataPart::loadIndex() for (size_t i = 0; i < marks_count; ++i) //-V756 for (size_t j = 0; j < key_size; ++j) - key_serializations[j]->deserializeBinary(*loaded_index[j], *index_file); + key_serializations[j]->deserializeBinary(*loaded_index[j], *index_file, {}); for (size_t i = 0; i < key_size; ++i) { @@ -1350,7 +1350,7 @@ void IMergeTreeDataPart::storeVersionMetadata(bool force) const if (!wasInvolvedInTransaction() && !force) return; - LOG_TEST(storage.log, "Writing version for {} (creation: {}, removal {})", name, version.creation_tid, version.removal_tid); + LOG_TEST(storage.log, "Writing version for {} (creation: {}, removal {}, creation csn {})", name, version.creation_tid, version.removal_tid, version.creation_csn); assert(storage.supportsTransactions()); if (!isStoredOnDisk()) @@ -1382,7 +1382,7 @@ void IMergeTreeDataPart::appendCSNToVersionMetadata(VersionMetadata::WhichCSN wh void IMergeTreeDataPart::appendRemovalTIDToVersionMetadata(bool clear) const { chassert(!version.creation_tid.isEmpty()); - chassert(version.removal_csn == 0); + chassert(version.removal_csn == 0 || (version.removal_csn == Tx::PrehistoricCSN && version.removal_tid.isPrehistoric())); chassert(!version.removal_tid.isEmpty()); chassert(isStoredOnDisk()); @@ -1390,6 +1390,12 @@ void IMergeTreeDataPart::appendRemovalTIDToVersionMetadata(bool clear) const { /// Metadata file probably does not exist, because it was not written on part creation, because it was created without a transaction. /// Let's create it (if needed). Concurrent writes are not possible, because creation_csn is prehistoric and we own removal_tid_lock. + + /// It can happen that VersionMetadata::isVisible sets creation_csn to PrehistoricCSN when creation_tid is Prehistoric + /// In order to avoid a race always write creation_csn as PrehistoricCSN for Prehistoric creation_tid + assert(version.creation_csn == Tx::UnknownCSN || version.creation_csn == Tx::PrehistoricCSN); + version.creation_csn.store(Tx::PrehistoricCSN); + storeVersionMetadata(); return; } @@ -1531,8 +1537,8 @@ bool IMergeTreeDataPart::assertHasValidVersionMetadata() const { WriteBufferFromOwnString expected; version.write(expected); - tryLogCurrentException(storage.log, fmt::format("File {} contains:\n{}\nexpected:\n{}\nlock: {}", - version_file_name, content, expected.str(), version.removal_tid_lock)); + tryLogCurrentException(storage.log, fmt::format("File {} contains:\n{}\nexpected:\n{}\nlock: {}\nname: {}", + version_file_name, content, expected.str(), version.removal_tid_lock, name)); return false; } } @@ -2023,8 +2029,7 @@ std::optional getIndexExtensionFromFilesystem(const IDataPartStorag for (auto it = data_part_storage.iterate(); it->isValid(); it->next()) { const auto & extension = fs::path(it->name()).extension(); - if (extension == getIndexExtension(false) - || extension == getIndexExtension(true)) + if (extension == getIndexExtension(true)) return extension; } } @@ -2036,4 +2041,12 @@ bool isCompressedFromIndexExtension(const String & index_extension) return index_extension == getIndexExtension(true); } +Strings getPartsNames(const MergeTreeDataPartsVector & parts) +{ + Strings part_names; + for (const auto & p : parts) + part_names.push_back(p->name); + return part_names; +} + } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 6515eb1a65c..e6c6f02b098 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -595,4 +595,8 @@ inline String getIndexExtension(bool is_compressed_primary_key) { return is_comp std::optional getIndexExtensionFromFilesystem(const IDataPartStorage & data_part_storage); bool isCompressedFromIndexExtension(const String & index_extension); +using MergeTreeDataPartsVector = std::vector; + +Strings getPartsNames(const MergeTreeDataPartsVector & parts); + } diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp index 02e61a70eb6..76d69cc6b7d 100644 --- a/src/Storages/MergeTree/MergeList.cpp +++ b/src/Storages/MergeTree/MergeList.cpp @@ -142,7 +142,11 @@ MergeInfo MergeListElement::getInfo() const return res; } -MergeListElement::~MergeListElement() = default; +MergeListElement::~MergeListElement() +{ + CurrentThread::getMemoryTracker()->adjustWithUntrackedMemory(untracked_memory); + untracked_memory = 0; +} } diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index d6cabddfec4..17a56272a57 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -74,8 +74,8 @@ private: MergeListEntry & merge_list_entry; MemoryTracker * background_thread_memory_tracker; MemoryTracker * background_thread_memory_tracker_prev_parent = nullptr; - UInt64 prev_untracked_memory_limit; - UInt64 prev_untracked_memory; + Int64 prev_untracked_memory_limit; + Int64 prev_untracked_memory; String prev_query_id; }; diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 227a5c2a0ca..b3185b1a6af 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -25,8 +25,20 @@ namespace ErrorCodes extern const int QUERY_WAS_CANCELLED; } +static void injectNonConstVirtualColumns( + size_t rows, + Block & block, + const Names & virtual_columns); -MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor( +static void injectPartConstVirtualColumns( + size_t rows, + Block & block, + MergeTreeReadTask * task, + const DataTypePtr & partition_value_type, + const Names & virtual_columns); + + +IMergeTreeSelectAlgorithm::IMergeTreeSelectAlgorithm( Block header, const MergeTreeData & storage_, const StorageSnapshotPtr & storage_snapshot_, @@ -39,8 +51,7 @@ MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor( bool use_uncompressed_cache_, const Names & virt_column_names_, std::optional extension_) - : ISource(transformHeader(std::move(header), prewhere_info_, storage_.getPartitionValueType(), virt_column_names_)) - , storage(storage_) + : storage(storage_) , storage_snapshot(storage_snapshot_) , prewhere_info(prewhere_info_) , prewhere_actions(getPrewhereActions(prewhere_info, actions_settings)) @@ -53,30 +64,20 @@ MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor( , partition_value_type(storage.getPartitionValueType()) , extension(extension_) { - header_without_virtual_columns = getPort().getHeader(); + header_without_const_virtual_columns = applyPrewhereActions(std::move(header), prewhere_info); + size_t non_const_columns_offset = header_without_const_virtual_columns.columns(); + injectNonConstVirtualColumns(0, header_without_const_virtual_columns, virt_column_names); /// Reverse order is to minimize reallocations when removing columns from the block - for (auto it = virt_column_names.rbegin(); it != virt_column_names.rend(); ++it) - { - if (*it == "_part_offset") - { - non_const_virtual_column_names.emplace_back(*it); - } - else if (*it == LightweightDeleteDescription::FILTER_COLUMN.name) - { - non_const_virtual_column_names.emplace_back(*it); - } - else - { - /// Remove virtual columns that are going to be filled with const values - if (header_without_virtual_columns.has(*it)) - header_without_virtual_columns.erase(*it); - } - } + for (size_t col_num = non_const_columns_offset; col_num < header_without_const_virtual_columns.columns(); ++col_num) + non_const_virtual_column_names.emplace_back(header_without_const_virtual_columns.getByPosition(col_num).name); + + result_header = header_without_const_virtual_columns; + injectPartConstVirtualColumns(0, result_header, nullptr, partition_value_type, virt_column_names); } -std::unique_ptr MergeTreeBaseSelectProcessor::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings) +std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings) { std::unique_ptr prewhere_actions; if (prewhere_info) @@ -111,7 +112,7 @@ std::unique_ptr MergeTreeBaseSelectProcessor::getPrewhereActio } -bool MergeTreeBaseSelectProcessor::getNewTask() +bool IMergeTreeSelectAlgorithm::getNewTask() { /// No parallel reading feature if (!extension.has_value()) @@ -127,7 +128,7 @@ bool MergeTreeBaseSelectProcessor::getNewTask() } -bool MergeTreeBaseSelectProcessor::getNewTaskParallelReading() +bool IMergeTreeSelectAlgorithm::getNewTaskParallelReading() { if (getTaskFromBuffer()) return true; @@ -152,7 +153,7 @@ bool MergeTreeBaseSelectProcessor::getNewTaskParallelReading() } -bool MergeTreeBaseSelectProcessor::getTaskFromBuffer() +bool IMergeTreeSelectAlgorithm::getTaskFromBuffer() { while (!buffered_ranges.empty()) { @@ -174,7 +175,7 @@ bool MergeTreeBaseSelectProcessor::getTaskFromBuffer() } -bool MergeTreeBaseSelectProcessor::getDelayedTasks() +bool IMergeTreeSelectAlgorithm::getDelayedTasks() { while (!delayed_tasks.empty()) { @@ -197,20 +198,23 @@ bool MergeTreeBaseSelectProcessor::getDelayedTasks() } -Chunk MergeTreeBaseSelectProcessor::generate() +ChunkAndProgress IMergeTreeSelectAlgorithm::read() { - while (!isCancelled()) + size_t num_read_rows = 0; + size_t num_read_bytes = 0; + + while (!is_cancelled) { try { if ((!task || task->isFinished()) && !getNewTask()) - return {}; + break; } catch (const Exception & e) { /// See MergeTreeBaseSelectProcessor::getTaskFromBuffer() if (e.code() == ErrorCodes::QUERY_WAS_CANCELLED) - return {}; + break; throw; } @@ -220,24 +224,35 @@ Chunk MergeTreeBaseSelectProcessor::generate() { injectVirtualColumns(res.block, res.row_count, task.get(), partition_value_type, virt_column_names); - /// Reorder the columns according to output header - const auto & output_header = output.getHeader(); + /// Reorder the columns according to result_header Columns ordered_columns; - ordered_columns.reserve(output_header.columns()); - for (size_t i = 0; i < output_header.columns(); ++i) + ordered_columns.reserve(result_header.columns()); + for (size_t i = 0; i < result_header.columns(); ++i) { - auto name = output_header.getByPosition(i).name; + auto name = result_header.getByPosition(i).name; ordered_columns.push_back(res.block.getByName(name).column); } - return Chunk(ordered_columns, res.row_count); + /// Account a progress from previous empty chunks. + res.num_read_rows += num_read_rows; + res.num_read_bytes += num_read_bytes; + + return ChunkAndProgress{ + .chunk = Chunk(ordered_columns, res.row_count), + .num_read_rows = res.num_read_rows, + .num_read_bytes = res.num_read_bytes}; + } + else + { + num_read_rows += res.num_read_rows; + num_read_bytes += res.num_read_bytes; } } - return {}; + return {Chunk(), num_read_rows, num_read_bytes}; } -void MergeTreeBaseSelectProcessor::initializeMergeTreeReadersForPart( +void IMergeTreeSelectAlgorithm::initializeMergeTreeReadersForPart( MergeTreeData::DataPartPtr & data_part, const MergeTreeReadTaskColumns & task_columns, const StorageMetadataPtr & metadata_snapshot, const MarkRanges & mark_ranges, const IMergeTreeReader::ValueSizeMap & value_size_map, @@ -268,7 +283,7 @@ void MergeTreeBaseSelectProcessor::initializeMergeTreeReadersForPart( } } -void MergeTreeBaseSelectProcessor::initializeRangeReaders(MergeTreeReadTask & current_task) +void IMergeTreeSelectAlgorithm::initializeRangeReaders(MergeTreeReadTask & current_task) { return initializeRangeReadersImpl( current_task.range_reader, current_task.pre_range_readers, prewhere_info, prewhere_actions.get(), @@ -276,7 +291,7 @@ void MergeTreeBaseSelectProcessor::initializeRangeReaders(MergeTreeReadTask & cu pre_reader_for_step, lightweight_delete_filter_step, non_const_virtual_column_names); } -void MergeTreeBaseSelectProcessor::initializeRangeReadersImpl( +void IMergeTreeSelectAlgorithm::initializeRangeReadersImpl( MergeTreeRangeReader & range_reader, std::deque & pre_range_readers, PrewhereInfoPtr prewhere_info, const PrewhereExprInfo * prewhere_actions, IMergeTreeReader * reader, bool has_lightweight_delete, const MergeTreeReaderSettings & reader_settings, @@ -368,7 +383,7 @@ static UInt64 estimateNumRows(const MergeTreeReadTask & current_task, UInt64 cur } -MergeTreeBaseSelectProcessor::BlockAndRowCount MergeTreeBaseSelectProcessor::readFromPartImpl() +IMergeTreeSelectAlgorithm::BlockAndProgress IMergeTreeSelectAlgorithm::readFromPartImpl() { if (task->size_predictor) task->size_predictor->startBlock(); @@ -398,7 +413,8 @@ MergeTreeBaseSelectProcessor::BlockAndRowCount MergeTreeBaseSelectProcessor::rea UInt64 num_filtered_rows = read_result.numReadRows() - read_result.num_rows; - progress(read_result.numReadRows(), read_result.numBytesRead()); + size_t num_read_rows = read_result.numReadRows(); + size_t num_read_bytes = read_result.numBytesRead(); if (task->size_predictor) { @@ -408,16 +424,21 @@ MergeTreeBaseSelectProcessor::BlockAndRowCount MergeTreeBaseSelectProcessor::rea task->size_predictor->update(sample_block, read_result.columns, read_result.num_rows); } - if (read_result.num_rows == 0) - return {}; + Block block; + if (read_result.num_rows != 0) + block = sample_block.cloneWithColumns(read_result.columns); - BlockAndRowCount res = { sample_block.cloneWithColumns(read_result.columns), read_result.num_rows }; + BlockAndProgress res = { + .block = std::move(block), + .row_count = read_result.num_rows, + .num_read_rows = num_read_rows, + .num_read_bytes = num_read_bytes }; return res; } -MergeTreeBaseSelectProcessor::BlockAndRowCount MergeTreeBaseSelectProcessor::readFromPart() +IMergeTreeSelectAlgorithm::BlockAndProgress IMergeTreeSelectAlgorithm::readFromPart() { if (!task->range_reader.isInitialized()) initializeRangeReaders(*task); @@ -474,9 +495,10 @@ namespace /// Adds virtual columns that are not const for all rows static void injectNonConstVirtualColumns( size_t rows, - VirtualColumnsInserter & inserter, + Block & block, const Names & virtual_columns) { + VirtualColumnsInserter inserter(block); for (const auto & virtual_column_name : virtual_columns) { if (virtual_column_name == "_part_offset") @@ -511,11 +533,12 @@ static void injectNonConstVirtualColumns( /// Adds virtual columns that are const for the whole part static void injectPartConstVirtualColumns( size_t rows, - VirtualColumnsInserter & inserter, + Block & block, MergeTreeReadTask * task, const DataTypePtr & partition_value_type, const Names & virtual_columns) { + VirtualColumnsInserter inserter(block); /// add virtual columns /// Except _sample_factor, which is added from the outside. if (!virtual_columns.empty()) @@ -584,19 +607,16 @@ static void injectPartConstVirtualColumns( } } -void MergeTreeBaseSelectProcessor::injectVirtualColumns( +void IMergeTreeSelectAlgorithm::injectVirtualColumns( Block & block, size_t row_count, MergeTreeReadTask * task, const DataTypePtr & partition_value_type, const Names & virtual_columns) { - VirtualColumnsInserter inserter{block}; - /// First add non-const columns that are filled by the range reader and then const columns that we will fill ourselves. /// Note that the order is important: virtual columns filled by the range reader must go first - injectNonConstVirtualColumns(row_count, inserter, virtual_columns); - injectPartConstVirtualColumns(row_count, inserter, task, partition_value_type, virtual_columns); + injectNonConstVirtualColumns(row_count, block, virtual_columns); + injectPartConstVirtualColumns(row_count, block, task, partition_value_type, virtual_columns); } -Block MergeTreeBaseSelectProcessor::transformHeader( - Block block, const PrewhereInfoPtr & prewhere_info, const DataTypePtr & partition_value_type, const Names & virtual_columns) +Block IMergeTreeSelectAlgorithm::applyPrewhereActions(Block block, const PrewhereInfoPtr & prewhere_info) { if (prewhere_info) { @@ -638,11 +658,18 @@ Block MergeTreeBaseSelectProcessor::transformHeader( } } - injectVirtualColumns(block, 0, nullptr, partition_value_type, virtual_columns); return block; } -std::unique_ptr MergeTreeBaseSelectProcessor::getSizePredictor( +Block IMergeTreeSelectAlgorithm::transformHeader( + Block block, const PrewhereInfoPtr & prewhere_info, const DataTypePtr & partition_value_type, const Names & virtual_columns) +{ + auto transformed = applyPrewhereActions(std::move(block), prewhere_info); + injectVirtualColumns(transformed, 0, nullptr, partition_value_type, virtual_columns); + return transformed; +} + +std::unique_ptr IMergeTreeSelectAlgorithm::getSizePredictor( const MergeTreeData::DataPartPtr & data_part, const MergeTreeReadTaskColumns & task_columns, const Block & sample_block) @@ -660,7 +687,7 @@ std::unique_ptr MergeTreeBaseSelectProcessor::getSi } -MergeTreeBaseSelectProcessor::Status MergeTreeBaseSelectProcessor::performRequestToCoordinator(MarkRanges requested_ranges, bool delayed) +IMergeTreeSelectAlgorithm::Status IMergeTreeSelectAlgorithm::performRequestToCoordinator(MarkRanges requested_ranges, bool delayed) { String partition_id = task->data_part->info.partition_id; String part_name; @@ -732,7 +759,7 @@ MergeTreeBaseSelectProcessor::Status MergeTreeBaseSelectProcessor::performReques } -size_t MergeTreeBaseSelectProcessor::estimateMaxBatchSizeForHugeRanges() +size_t IMergeTreeSelectAlgorithm::estimateMaxBatchSizeForHugeRanges() { /// This is an empirical number and it is so, /// because we have an adaptive granularity by default. @@ -768,7 +795,7 @@ size_t MergeTreeBaseSelectProcessor::estimateMaxBatchSizeForHugeRanges() return max_size_for_one_request / sum_average_marks_size; } -void MergeTreeBaseSelectProcessor::splitCurrentTaskRangesAndFillBuffer() +void IMergeTreeSelectAlgorithm::splitCurrentTaskRangesAndFillBuffer() { const size_t max_batch_size = estimateMaxBatchSizeForHugeRanges(); @@ -824,6 +851,6 @@ void MergeTreeBaseSelectProcessor::splitCurrentTaskRangesAndFillBuffer() buffered_ranges.pop_back(); } -MergeTreeBaseSelectProcessor::~MergeTreeBaseSelectProcessor() = default; +IMergeTreeSelectAlgorithm::~IMergeTreeSelectAlgorithm() = default; } diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h index e385f5f4d25..2a5f6871422 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h @@ -1,12 +1,10 @@ #pragma once - #include #include #include #include #include - -#include +#include namespace DB @@ -17,6 +15,12 @@ class UncompressedCache; class MarkCache; struct PrewhereExprInfo; +struct ChunkAndProgress +{ + Chunk chunk; + size_t num_read_rows = 0; + size_t num_read_bytes = 0; +}; struct ParallelReadingExtension { @@ -29,11 +33,11 @@ struct ParallelReadingExtension Names colums_to_read; }; -/// Base class for MergeTreeThreadSelectProcessor and MergeTreeSelectProcessor -class MergeTreeBaseSelectProcessor : public ISource +/// Base class for MergeTreeThreadSelectAlgorithm and MergeTreeSelectAlgorithm +class IMergeTreeSelectAlgorithm { public: - MergeTreeBaseSelectProcessor( + IMergeTreeSelectAlgorithm( Block header, const MergeTreeData & storage_, const StorageSnapshotPtr & storage_snapshot_, @@ -47,7 +51,7 @@ public: const Names & virt_column_names_ = {}, std::optional extension_ = {}); - ~MergeTreeBaseSelectProcessor() override; + virtual ~IMergeTreeSelectAlgorithm(); static Block transformHeader( Block block, const PrewhereInfoPtr & prewhere_info, const DataTypePtr & partition_value_type, const Names & virtual_columns); @@ -57,16 +61,26 @@ public: const MergeTreeReadTaskColumns & task_columns, const Block & sample_block); + Block getHeader() const { return result_header; } + + ChunkAndProgress read(); + + void cancel() { is_cancelled = true; } + + const MergeTreeReaderSettings & getSettings() const { return reader_settings; } + + virtual std::string getName() const = 0; + protected: /// This struct allow to return block with no columns but with non-zero number of rows similar to Chunk - struct BlockAndRowCount + struct BlockAndProgress { Block block; size_t row_count = 0; + size_t num_read_rows = 0; + size_t num_read_bytes = 0; }; - Chunk generate() final; - /// Creates new this->task and return a flag whether it was successful or not virtual bool getNewTaskImpl() = 0; /// Creates new readers for a task it is needed. These methods are separate, because @@ -81,9 +95,9 @@ protected: /// Closes readers and unlock part locks virtual void finish() = 0; - virtual BlockAndRowCount readFromPart(); + virtual BlockAndProgress readFromPart(); - BlockAndRowCount readFromPartImpl(); + BlockAndProgress readFromPartImpl(); /// Used for filling header with no rows as well as block with data static void @@ -137,7 +151,9 @@ protected: DataTypePtr partition_value_type; /// This header is used for chunks from readFromPart(). - Block header_without_virtual_columns; + Block header_without_const_virtual_columns; + /// A result of getHeader(). A chunk which this header is returned from read(). + Block result_header; std::shared_ptr owned_uncompressed_cache; std::shared_ptr owned_mark_cache; @@ -156,6 +172,8 @@ protected: private: Poco::Logger * log = &Poco::Logger::get("MergeTreeBaseSelectProcessor"); + std::atomic is_cancelled{false}; + enum class Status { Accepted, @@ -194,6 +212,9 @@ private: Status performRequestToCoordinator(MarkRanges requested_ranges, bool delayed); void splitCurrentTaskRangesAndFillBuffer(); + static Block applyPrewhereActions(Block block, const PrewhereInfoPtr & prewhere_info); }; +using MergeTreeSelectAlgorithmPtr = std::unique_ptr; + } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 387a6388f64..7ff8d95b52a 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,7 @@ #include #include #include +#include #include #include @@ -168,6 +170,7 @@ namespace ErrorCodes extern const int INCORRECT_QUERY; extern const int CANNOT_RESTORE_TABLE; extern const int ZERO_COPY_REPLICATION_ERROR; + extern const int SERIALIZATION_ERROR; } @@ -1147,6 +1150,10 @@ void MergeTreeData::loadDataPartsFromDisk( { pool.scheduleOrThrowOnError([&, thread, thread_group = CurrentThread::getGroup()] { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); if (thread_group) CurrentThread::attachToIfDetached(thread_group); @@ -1695,7 +1702,7 @@ size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lif return cleared_count; } -scope_guard MergeTreeData::getTemporaryPartDirectoryHolder(const String & part_dir_name) +scope_guard MergeTreeData::getTemporaryPartDirectoryHolder(const String & part_dir_name) const { temporary_parts.add(part_dir_name); return [this, part_dir_name]() { temporary_parts.remove(part_dir_name); }; @@ -1725,6 +1732,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) /// in the "zero-copy replication" (because it is a non-production feature). /// Please don't use "zero-copy replication" (a non-production feature) in production. /// It is not ready for production usage. Don't use it. + bool need_remove_parts_in_order = supportsReplication() && getSettings()->allow_remote_fs_zero_copy_replication; if (need_remove_parts_in_order) @@ -1741,7 +1749,6 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) need_remove_parts_in_order = has_zero_copy_disk; } - time_t now = time(nullptr); std::vector parts_to_delete; std::vector skipped_parts; @@ -1757,6 +1764,8 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) return false; }; + auto time_now = time(nullptr); + { auto parts_lock = lockParts(); @@ -1772,8 +1781,6 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) continue; } - auto part_remove_time = part->remove_time.load(std::memory_order_relaxed); - /// Grab only parts that are not used by anyone (SELECTs for example). if (!part.unique()) { @@ -1781,7 +1788,8 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) continue; } - if ((part_remove_time < now && now - part_remove_time > getSettings()->old_parts_lifetime.totalSeconds() && !has_skipped_mutation_parent(part)) + auto part_remove_time = part->remove_time.load(std::memory_order_relaxed); + if ((part_remove_time < time_now && time_now - part_remove_time > getSettings()->old_parts_lifetime.totalSeconds() && !has_skipped_mutation_parent(part)) || force || isInMemoryPart(part) /// Remove in-memory parts immediately to not store excessive data in RAM || (part->version.creation_csn == Tx::RolledBackCSN && getSettings()->remove_rolled_back_parts_immediately)) @@ -1791,6 +1799,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) else { skipped_parts.push_back(part->info); + continue; } } @@ -1803,7 +1812,8 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) } if (!res.empty()) - LOG_TRACE(log, "Found {} old parts to remove.", res.size()); + LOG_TRACE(log, "Found {} old parts to remove. Parts {}", + res.size(), fmt::join(getPartsNames(res), ", ")); return res; } @@ -1838,6 +1848,8 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa (*it)->assertState({DataPartState::Deleting}); + LOG_DEBUG(log, "Finally removing part from memory {}", part->name); + data_parts_indexes.erase(it); } } @@ -1934,6 +1946,8 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts, bool { get_failed_parts(); + LOG_DEBUG(log, "Failed to remove all parts, all count {}, removed {}", parts.size(), part_names_succeed.size()); + if (throw_on_error) throw; } @@ -1967,6 +1981,10 @@ void MergeTreeData::clearPartsFromFilesystemImpl(const DataPartsVector & parts_t { pool.scheduleOrThrowOnError([&, thread_group = CurrentThread::getGroup()] { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); if (thread_group) CurrentThread::attachToIfDetached(thread_group); @@ -2124,11 +2142,24 @@ size_t MergeTreeData::clearEmptyParts() if (part->rows_count != 0) continue; - /// Do not try to drop uncommitted parts. + /// Do not try to drop uncommitted parts. If the newest tx doesn't see it that is probably hasn't been committed jet if (!part->version.getCreationTID().isPrehistoric() && !part->version.isVisible(TransactionLog::instance().getLatestSnapshot())) continue; - LOG_TRACE(log, "Will drop empty part {}", part->name); + /// Don't drop empty parts that cover other parts + /// Otherwise covered parts resurrect + { + auto lock = lockParts(); + if (part->getState() != DataPartState::Active) + continue; + + DataPartsVector covered_parts = getCoveredOutdatedParts(part, lock); + if (!covered_parts.empty()) + continue; + } + + LOG_INFO(log, "Will drop empty part {}", part->name); + dropPartNoWaitNoThrow(part->name); ++cleared_count; } @@ -2905,16 +2936,16 @@ MergeTreeData::PartsTemporaryRename::~PartsTemporaryRename() } } - -MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( - const MergeTreePartInfo & new_part_info, - const String & new_part_name, - DataPartPtr & out_covering_part, +MergeTreeData::PartHierarchy MergeTreeData::getPartHierarchy( + const MergeTreePartInfo & part_info, + DataPartState state, DataPartsLock & /* data_parts_lock */) const { + PartHierarchy result; + /// Parts contained in the part are consecutive in data_parts, intersecting the insertion place for the part itself. - auto it_middle = data_parts_by_state_and_info.lower_bound(DataPartStateAndInfo{DataPartState::Active, new_part_info}); - auto committed_parts_range = getDataPartsStateRange(DataPartState::Active); + auto it_middle = data_parts_by_state_and_info.lower_bound(DataPartStateAndInfo{state, part_info}); + auto committed_parts_range = getDataPartsStateRange(state); /// Go to the left. DataPartIteratorByStateAndInfo begin = it_middle; @@ -2922,17 +2953,16 @@ MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( { auto prev = std::prev(begin); - if (!new_part_info.contains((*prev)->info)) + if (!part_info.contains((*prev)->info)) { - if ((*prev)->info.contains(new_part_info)) + if ((*prev)->info.contains(part_info)) { - out_covering_part = *prev; - return {}; + result.covering_parts.push_back(*prev); + } + else if (!part_info.isDisjoint((*prev)->info)) + { + result.intersected_parts.push_back(*prev); } - - if (!new_part_info.isDisjoint((*prev)->info)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} intersects previous part {}. It is a bug.", - new_part_name, (*prev)->getNameWithState()); break; } @@ -2940,24 +2970,29 @@ MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( begin = prev; } + std::reverse(result.covering_parts.begin(), result.covering_parts.end()); + /// Go to the right. DataPartIteratorByStateAndInfo end = it_middle; while (end != committed_parts_range.end()) { - if ((*end)->info == new_part_info) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected duplicate part {}. It is a bug.", (*end)->getNameWithState()); - - if (!new_part_info.contains((*end)->info)) + if ((*end)->info == part_info) { - if ((*end)->info.contains(new_part_info)) - { - out_covering_part = *end; - return {}; - } + result.duplicate_part = *end; + result.covering_parts.clear(); + return result; + } - if (!new_part_info.isDisjoint((*end)->info)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} intersects next part {}. It is a bug.", - new_part_name, (*end)->getNameWithState()); + if (!part_info.contains((*end)->info)) + { + if ((*end)->info.contains(part_info)) + { + result.covering_parts.push_back(*end); + } + else if (!part_info.isDisjoint((*end)->info)) + { + result.intersected_parts.push_back(*end); + } break; } @@ -2965,31 +3000,47 @@ MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( ++end; } - return DataPartsVector{begin, end}; + result.covered_parts.insert(result.covered_parts.end(), begin, end); + + return result; } - -bool MergeTreeData::renameTempPartAndAdd( - MutableDataPartPtr & part, - Transaction & out_transaction, - DataPartsLock & lock) +MergeTreeData::DataPartsVector MergeTreeData::getCoveredOutdatedParts( + const DataPartPtr & part, + DataPartsLock & data_parts_lock) const { - DataPartsVector covered_parts; + part->assertState({DataPartState::Active, DataPartState::PreActive}); + PartHierarchy hierarchy = getPartHierarchy(part->info, DataPartState::Outdated, data_parts_lock); - if (!renameTempPartAndReplaceImpl(part, out_transaction, lock, &covered_parts)) - return false; + if (hierarchy.duplicate_part) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected duplicate part {}. It is a bug.", hierarchy.duplicate_part->getNameWithState()); - if (!covered_parts.empty()) - throw Exception("Added part " + part->name + " covers " + toString(covered_parts.size()) - + " existing part(s) (including " + covered_parts[0]->name + ")", ErrorCodes::LOGICAL_ERROR); - - return true; + return hierarchy.covered_parts; } -void MergeTreeData::checkPartCanBeAddedToTable(MutableDataPartPtr & part, DataPartsLock & lock) const +MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( + const MergeTreePartInfo & new_part_info, + const String & new_part_name, + DataPartPtr & out_covering_part, + DataPartsLock & data_parts_lock) const { - part->assertState({DataPartState::Temporary}); + PartHierarchy hierarchy = getPartHierarchy(new_part_info, DataPartState::Active, data_parts_lock); + if (!hierarchy.intersected_parts.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} intersects part {}. It is a bug.", + new_part_name, hierarchy.intersected_parts.back()->getNameWithState()); + + if (hierarchy.duplicate_part) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected duplicate part {}. It is a bug.", hierarchy.duplicate_part->getNameWithState()); + + if (!hierarchy.covering_parts.empty()) + out_covering_part = std::move(hierarchy.covering_parts.back()); + + return std::move(hierarchy.covered_parts); +} + +void MergeTreeData::checkPartPartition(MutableDataPartPtr & part, DataPartsLock & lock) const +{ if (DataPartPtr existing_part_in_partition = getAnyPartInPartition(part->info.partition_id, lock)) { if (part->partition.value != existing_part_in_partition->partition.value) @@ -2998,14 +3049,22 @@ void MergeTreeData::checkPartCanBeAddedToTable(MutableDataPartPtr & part, DataPa + existing_part_in_partition->name + ", newly added part: " + part->name, ErrorCodes::CORRUPTED_DATA); } +} - if (auto it_duplicate = data_parts_by_info.find(part->info); it_duplicate != data_parts_by_info.end()) +void MergeTreeData::checkPartDuplicate(MutableDataPartPtr & part, Transaction & transaction, DataPartsLock & /*lock*/) const +{ + auto it_duplicate = data_parts_by_info.find(part->info); + + if (it_duplicate != data_parts_by_info.end()) { String message = "Part " + (*it_duplicate)->getNameWithState() + " already exists"; if ((*it_duplicate)->checkState({DataPartState::Outdated, DataPartState::Deleting})) throw Exception(message + ", but it will be deleted soon", ErrorCodes::PART_IS_TEMPORARILY_LOCKED); + if (transaction.txn) + throw Exception(message, ErrorCodes::SERIALIZATION_ERROR); + throw Exception(message, ErrorCodes::DUPLICATE_DATA_PART); } } @@ -3034,49 +3093,59 @@ bool MergeTreeData::renameTempPartAndReplaceImpl( DataPartsLock & lock, DataPartsVector * out_covered_parts) { - LOG_TRACE(log, "Renaming temporary part {} to {}.", part->getDataPartStorage().getPartDirectory(), part->name); + LOG_TRACE(log, "Renaming temporary part {} to {} with tid {}.", part->getDataPartStorage().getPartDirectory(), part->name, out_transaction.getTID()); if (&out_transaction.data != this) - throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", - ErrorCodes::LOGICAL_ERROR); + throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", ErrorCodes::LOGICAL_ERROR); + + part->assertState({DataPartState::Temporary}); + checkPartPartition(part, lock); + checkPartDuplicate(part, out_transaction, lock); + + PartHierarchy hierarchy = getPartHierarchy(part->info, DataPartState::Active, lock); + + if (!hierarchy.intersected_parts.empty()) + { + String message = fmt::format("Part {} intersects part {}", part->name, hierarchy.intersected_parts.back()->getNameWithState()); + + // Drop part|partition operation inside some transactions sees some stale snapshot from the time when transactions has been started. + // So such operation may attempt to delete already outdated part. In this case, this outdated part is most likely covered by the other part and intersection may occur. + // Part mayght be outdated due to merge|mutation|update|optimization operations. + if (part->isEmpty() || (hierarchy.intersected_parts.size() == 1 && hierarchy.intersected_parts.back()->isEmpty())) + { + message += fmt::format(" One of them is empty part. That is a race between drop operation under transaction and a merge/mutation."); + throw Exception(message, ErrorCodes::SERIALIZATION_ERROR); + } + + if (hierarchy.intersected_parts.size() > 1) + message += fmt::format(" There are {} intersected parts.", hierarchy.intersected_parts.size()); + + throw Exception(ErrorCodes::LOGICAL_ERROR, message + " It is a bug."); + } if (part->hasLightweightDelete()) has_lightweight_delete_parts.store(true); - checkPartCanBeAddedToTable(part, lock); - - DataPartPtr covering_part; - DataPartsVector covered_parts = getActivePartsToReplace(part->info, part->name, covering_part, lock); - - if (covering_part) - { - LOG_WARNING(log, "Tried to add obsolete part {} covered by {}", part->name, covering_part->getNameWithState()); - return false; - } - /// All checks are passed. Now we can rename the part on disk. /// So, we maintain invariant: if a non-temporary part in filesystem then it is in data_parts preparePartForCommit(part, out_transaction); if (out_covered_parts) { - out_covered_parts->reserve(covered_parts.size()); - - for (DataPartPtr & covered_part : covered_parts) - out_covered_parts->emplace_back(std::move(covered_part)); + out_covered_parts->reserve(out_covered_parts->size() + hierarchy.covered_parts.size()); + std::move(hierarchy.covered_parts.begin(), hierarchy.covered_parts.end(), std::back_inserter(*out_covered_parts)); } return true; } -MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplaceUnlocked( +bool MergeTreeData::renameTempPartAndReplaceUnlocked( MutableDataPartPtr & part, Transaction & out_transaction, - DataPartsLock & lock) + DataPartsLock & lock, + DataPartsVector * out_covered_parts) { - DataPartsVector covered_parts; - renameTempPartAndReplaceImpl(part, out_transaction, lock, &covered_parts); - return covered_parts; + return renameTempPartAndReplaceImpl(part, out_transaction, lock, out_covered_parts); } MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( @@ -3084,7 +3153,26 @@ MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( Transaction & out_transaction) { auto part_lock = lockParts(); - return renameTempPartAndReplaceUnlocked(part, out_transaction, part_lock); + DataPartsVector covered_parts; + renameTempPartAndReplaceImpl(part, out_transaction, part_lock, &covered_parts); + return covered_parts; +} + +bool MergeTreeData::renameTempPartAndAdd( + MutableDataPartPtr & part, + Transaction & out_transaction, + DataPartsLock & lock) +{ + DataPartsVector covered_parts; + + if (!renameTempPartAndReplaceImpl(part, out_transaction, lock, &covered_parts)) + return false; + + if (!covered_parts.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Added part {} covers {} existing part(s) (including {})", + part->name, toString(covered_parts.size()), covered_parts[0]->name); + + return true; } void MergeTreeData::removePartsFromWorkingSet(MergeTreeTransaction * txn, const MergeTreeData::DataPartsVector & remove, bool clear_without_timeout, DataPartsLock & acquired_lock) @@ -4561,17 +4649,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr loc DataPartsVector MergeTreeData::getVisibleDataPartsVector(ContextPtr local_context) const { - DataPartsVector res; - if (const auto * txn = local_context->getCurrentTransaction().get()) - { - res = getDataPartsVectorForInternalUsage({DataPartState::Active, DataPartState::Outdated}); - filterVisibleDataParts(res, txn->getSnapshot(), txn->tid); - } - else - { - res = getDataPartsVectorForInternalUsage(); - } - return res; + return getVisibleDataPartsVector(local_context->getCurrentTransaction()); } DataPartsVector MergeTreeData::getVisibleDataPartsVectorUnlocked(ContextPtr local_context, const DataPartsLock & lock) const @@ -4623,17 +4701,8 @@ void MergeTreeData::filterVisibleDataParts(DataPartsVector & maybe_visible_parts std::erase_if(maybe_visible_parts, need_remove_pred); [[maybe_unused]] size_t visible_size = maybe_visible_parts.size(); - - auto get_part_names = [&maybe_visible_parts]() -> Strings - { - Strings visible_part_names; - for (const auto & p : maybe_visible_parts) - visible_part_names.push_back(p->name); - return visible_part_names; - }; - LOG_TEST(log, "Got {} parts (of {}) visible in snapshot {} (TID {}): {}", - visible_size, total_size, snapshot_version, current_tid, fmt::join(get_part_names(), ", ")); + visible_size, total_size, snapshot_version, current_tid, fmt::join(getPartsNames(maybe_visible_parts), ", ")); } @@ -4666,6 +4735,22 @@ std::set MergeTreeData::getPartitionIdsAffectedByCommands( return affected_partition_ids; } +std::unordered_set MergeTreeData::getAllPartitionIds() const +{ + auto lock = lockParts(); + std::unordered_set res; + std::string_view prev_id; + for (const auto & part : getDataPartsStateRange(DataPartState::Active)) + { + if (prev_id == part->info.partition_id) + continue; + + res.insert(part->info.partition_id); + prev_id = part->info.partition_id; + } + return res; +} + MergeTreeData::DataPartsVector MergeTreeData::getDataPartsVectorForInternalUsage( const DataPartStates & affordable_states, const DataPartsLock & /*lock*/, DataPartStateVector * out_states) const @@ -5120,6 +5205,7 @@ CompressionCodecPtr MergeTreeData::getCompressionCodecForPart(size_t part_size_c static_cast(part_size_compressed) / getTotalActiveSizeInBytes()); } + MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const { DataParts res; @@ -5182,11 +5268,16 @@ void MergeTreeData::Transaction::rollbackPartsToTemporaryState() clear(); } +TransactionID MergeTreeData::Transaction::getTID() const +{ + if (txn) + return txn->tid; + return Tx::PrehistoricTID; +} + void MergeTreeData::Transaction::addPart(MutableDataPartPtr & part) { precommitted_parts.insert(part); - if (asInMemoryPart(part)) - has_in_memory_parts = true; } void MergeTreeData::Transaction::rollback() @@ -5194,11 +5285,14 @@ void MergeTreeData::Transaction::rollback() if (!isEmpty()) { WriteBufferFromOwnString buf; - buf << " Removing parts:"; + buf << "Removing parts:"; for (const auto & part : precommitted_parts) buf << " " << part->getDataPartStorage().getPartDirectory(); buf << "."; - LOG_DEBUG(data.log, "Undoing transaction.{}", buf.str()); + LOG_DEBUG(data.log, "Undoing transaction {}. {}", getTID(), buf.str()); + + for (const auto & part : precommitted_parts) + part->version.creation_csn.store(Tx::RolledBackCSN); auto lock = data.lockParts(); @@ -5229,7 +5323,6 @@ void MergeTreeData::Transaction::rollback() void MergeTreeData::Transaction::clear() { precommitted_parts.clear(); - has_in_memory_parts = false; } MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData::DataPartsLock * acquired_parts_lock) @@ -5246,26 +5339,41 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: if (part->getDataPartStorage().hasActiveTransaction()) part->getDataPartStorage().commitTransaction(); - bool commit_to_wal = has_in_memory_parts && settings->in_memory_parts_enable_wal; - if (txn || commit_to_wal) - { - MergeTreeData::WriteAheadLogPtr wal; - if (commit_to_wal) - wal = data.getWriteAheadLog(); - + if (txn) for (const auto & part : precommitted_parts) { - if (txn) - { - DataPartPtr covering_part; - DataPartsVector covered_parts = data.getActivePartsToReplace(part->info, part->name, covering_part, *owing_parts_lock); - MergeTreeTransaction::addNewPartAndRemoveCovered(data.shared_from_this(), part, covered_parts, txn); - } + DataPartPtr covering_part; + DataPartsVector covered_active_parts = data.getActivePartsToReplace(part->info, part->name, covering_part, *owing_parts_lock); - if (auto part_in_memory = asInMemoryPart(part)) - wal->addPart(part_in_memory); + /// outdated parts should be also collected here + /// the visible outdated parts should be tried to be removed + /// more likely the conflict happens at the removing visible outdated parts, what is right actually + DataPartsVector covered_outdated_parts = data.getCoveredOutdatedParts(part, *owing_parts_lock); + + LOG_TEST(data.log, "Got {} oudated parts covered by {} (TID {} CSN {}): {}", + covered_outdated_parts.size(), part->getNameWithState(), txn->tid, txn->getSnapshot(), fmt::join(getPartsNames(covered_outdated_parts), ", ")); + data.filterVisibleDataParts(covered_outdated_parts, txn->getSnapshot(), txn->tid); + + DataPartsVector covered_parts; + covered_parts.reserve(covered_active_parts.size() + covered_outdated_parts.size()); + std::move(covered_active_parts.begin(), covered_active_parts.end(), std::back_inserter(covered_parts)); + std::move(covered_outdated_parts.begin(), covered_outdated_parts.end(), std::back_inserter(covered_parts)); + + MergeTreeTransaction::addNewPartAndRemoveCovered(data.shared_from_this(), part, covered_parts, txn); } - } + + MergeTreeData::WriteAheadLogPtr wal; + auto get_inited_wal = [&] () + { + if (!wal) + wal = data.getWriteAheadLog(); + return wal; + }; + + if (settings->in_memory_parts_enable_wal) + for (const auto & part : precommitted_parts) + if (auto part_in_memory = asInMemoryPart(part)) + get_inited_wal()->addPart(part_in_memory); NOEXCEPT_SCOPE({ auto current_time = time(nullptr); @@ -5310,6 +5418,10 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: data.modifyPartState(covered_part, DataPartState::Outdated); data.removePartContributionToColumnAndSecondaryIndexSizes(covered_part); + + if (settings->in_memory_parts_enable_wal) + if (isInMemoryPart(covered_part)) + get_inited_wal()->dropPart(covered_part->name); } reduce_parts += covered_parts.size(); @@ -6292,24 +6404,38 @@ std::pair MergeTreeData::cloneAn auto reservation = src_part->getDataPartStorage().reserve(src_part->getBytesOnDisk()); auto src_part_storage = src_part->getDataPartStoragePtr(); + scope_guard src_flushed_tmp_dir_lock; + MergeTreeData::MutableDataPartPtr src_flushed_tmp_part; + /// If source part is in memory, flush it to disk and clone it already in on-disk format + /// Protect tmp dir from removing by cleanup thread with src_flushed_tmp_dir_lock + /// Construct src_flushed_tmp_part in order to delete part with its directory at destructor if (auto src_part_in_memory = asInMemoryPart(src_part)) { - auto flushed_part_path = src_part_in_memory->getRelativePathForPrefix(tmp_part_prefix); - src_part_storage = src_part_in_memory->flushToDisk(*flushed_part_path, metadata_snapshot); + auto flushed_part_path = *src_part_in_memory->getRelativePathForPrefix(tmp_part_prefix); + + auto tmp_src_part_file_name = fs::path(tmp_dst_part_name).filename(); + src_flushed_tmp_dir_lock = src_part->storage.getTemporaryPartDirectoryHolder(tmp_src_part_file_name); + + auto flushed_part_storage = src_part_in_memory->flushToDisk(flushed_part_path, metadata_snapshot); + src_flushed_tmp_part = createPart(src_part->name, src_part->info, flushed_part_storage); + src_flushed_tmp_part->is_temp = true; + + src_part_storage = flushed_part_storage; } String with_copy; if (copy_instead_of_hardlink) with_copy = " (copying data)"; - LOG_DEBUG(log, "Cloning part {} to {}{}", - src_part_storage->getFullPath(), - std::string(fs::path(src_part_storage->getFullRootPath()) / tmp_dst_part_name), - with_copy); - auto dst_part_storage = src_part_storage->freeze(relative_data_path, tmp_dst_part_name, /* make_source_readonly */ false, {}, copy_instead_of_hardlink, files_to_copy_instead_of_hardlinks); + LOG_DEBUG(log, "Clone {} part {} to {}{}", + src_flushed_tmp_part ? "flushed" : "", + src_part_storage->getFullPath(), + std::string(fs::path(dst_part_storage->getFullRootPath()) / tmp_dst_part_name), + with_copy); + auto dst_data_part = createPart(dst_part_name, dst_part_info, dst_part_storage); if (!copy_instead_of_hardlink && hardlinked_files) @@ -6322,7 +6448,25 @@ std::pair MergeTreeData::cloneAn if (!files_to_copy_instead_of_hardlinks.contains(it->name()) && it->name() != IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME && it->name() != IMergeTreeDataPart::TXN_VERSION_METADATA_FILE_NAME) + { hardlinked_files->hardlinks_from_source_part.insert(it->name()); + } + } + + auto projections = src_part->getProjectionParts(); + for (const auto & [name, projection_part] : projections) + { + const auto & projection_storage = projection_part->getDataPartStorage(); + for (auto it = projection_storage.iterate(); it->isValid(); it->next()) + { + auto file_name_with_projection_prefix = fs::path(projection_storage.getPartDirectory()) / it->name(); + if (!files_to_copy_instead_of_hardlinks.contains(file_name_with_projection_prefix) + && it->name() != IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME + && it->name() != IMergeTreeDataPart::TXN_VERSION_METADATA_FILE_NAME) + { + hardlinked_files->hardlinks_from_source_part.insert(file_name_with_projection_prefix); + } + } } } @@ -6475,12 +6619,21 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( LOG_DEBUG(log, "Freezing part {} snapshot will be placed at {}", part->name, backup_path); auto data_part_storage = part->getDataPartStoragePtr(); - String src_part_path = data_part_storage->getRelativePath(); String backup_part_path = fs::path(backup_path) / relative_data_path; + + scope_guard src_flushed_tmp_dir_lock; + MergeTreeData::MutableDataPartPtr src_flushed_tmp_part; + if (auto part_in_memory = asInMemoryPart(part)) { - auto flushed_part_path = part_in_memory->getRelativePathForPrefix("tmp_freeze"); - data_part_storage = part_in_memory->flushToDisk(*flushed_part_path, metadata_snapshot); + auto flushed_part_path = *part_in_memory->getRelativePathForPrefix("tmp_freeze"); + src_flushed_tmp_dir_lock = part->storage.getTemporaryPartDirectoryHolder("tmp_freeze" + part->name); + + auto flushed_part_storage = part_in_memory->flushToDisk(flushed_part_path, metadata_snapshot); + src_flushed_tmp_part = createPart(part->name, part->info, flushed_part_storage); + src_flushed_tmp_part->is_temp = true; + + data_part_storage = flushed_part_storage; } auto callback = [this, &part, &backup_part_path](const DiskPtr & disk) @@ -6565,6 +6718,7 @@ bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const if (canUseAdaptiveGranularity() && !src_part->index_granularity_info.mark_type.adaptive) return false; } + return true; } @@ -7234,6 +7388,89 @@ void MergeTreeData::incrementMergedPartsProfileEvent(MergeTreeDataPartType type) } } +MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart( + MergeTreePartInfo & new_part_info, const MergeTreePartition & partition, const String & new_part_name, + const MergeTreeTransactionPtr & txn) +{ + auto metadata_snapshot = getInMemoryMetadataPtr(); + auto settings = getSettings(); + + auto block = metadata_snapshot->getSampleBlock(); + NamesAndTypesList columns = metadata_snapshot->getColumns().getAllPhysical().filter(block.getNames()); + setAllObjectsToDummyTupleType(columns); + + auto minmax_idx = std::make_shared(); + minmax_idx->update(block, getMinMaxColumnsNames(metadata_snapshot->getPartitionKey())); + + DB::IMergeTreeDataPart::TTLInfos move_ttl_infos; + VolumePtr volume = getStoragePolicy()->getVolume(0); + ReservationPtr reservation = reserveSpacePreferringTTLRules(metadata_snapshot, 0, move_ttl_infos, time(nullptr), 0, true); + VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume); + + auto new_data_part_storage = std::make_shared( + data_part_volume, + getRelativeDataPath(), + EMPTY_PART_TMP_PREFIX + new_part_name); + + auto new_data_part = createPart( + new_part_name, + choosePartTypeOnDisk(0, block.rows()), + new_part_info, + new_data_part_storage + ); + + new_data_part->name = new_part_name; + + if (settings->assign_part_uuids) + new_data_part->uuid = UUIDHelpers::generateV4(); + + new_data_part->setColumns(columns, {}); + new_data_part->rows_count = block.rows(); + + new_data_part->partition = partition; + + new_data_part->minmax_idx = std::move(minmax_idx); + new_data_part->is_temp = true; + + SyncGuardPtr sync_guard; + if (new_data_part->isStoredOnDisk()) + { + /// The name could be non-unique in case of stale files from previous runs. + if (new_data_part_storage->exists()) + { + /// The path has to be unique, all tmp directories are deleted at startup in case of stale files from previous runs. + /// New part have to capture its name, therefore there is no concurrentcy in directory creation + throw Exception(ErrorCodes::LOGICAL_ERROR, + "New empty part is about to matirialize but the dirrectory already exist" + ", new part {}" + ", directory {}", + new_part_name, new_data_part_storage->getFullPath()); + } + + new_data_part_storage->createDirectories(); + + if (getSettings()->fsync_part_directory) + sync_guard = new_data_part_storage->getDirectorySyncGuard(); + } + + /// This effectively chooses minimal compression method: + /// either default lz4 or compression method with zero thresholds on absolute and relative part size. + auto compression_codec = getContext()->chooseCompressionCodec(0, 0); + + const auto & index_factory = MergeTreeIndexFactory::instance(); + MergedBlockOutputStream out(new_data_part, metadata_snapshot, columns, + index_factory.getMany(metadata_snapshot->getSecondaryIndices()), compression_codec, txn); + + bool sync_on_insert = settings->fsync_after_insert; + + out.write(block); + /// Here is no projections as no data inside + + out.finalizePart(new_data_part, sync_on_insert); + + return new_data_part; +} + CurrentlySubmergingEmergingTagger::~CurrentlySubmergingEmergingTagger() { std::lock_guard lock(storage.currently_submerging_emerging_mutex); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 8bd0fc1f280..02303031baa 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -29,7 +30,6 @@ #include #include #include -#include #include @@ -220,6 +220,9 @@ public: using DataPartsLock = std::unique_lock; DataPartsLock lockParts() const { return DataPartsLock(data_parts_mutex); } + using OperationDataPartsLock = std::unique_lock; + OperationDataPartsLock lockOperationsWithParts() const { return OperationDataPartsLock(operation_with_data_parts_mutex); } + MergeTreeDataPartType choosePartType(size_t bytes_uncompressed, size_t rows_count) const; MergeTreeDataPartType choosePartTypeOnDisk(size_t bytes_uncompressed, size_t rows_count) const; @@ -271,6 +274,8 @@ public: } } + TransactionID getTID() const; + private: friend class MergeTreeData; @@ -278,7 +283,6 @@ public: MergeTreeTransaction * txn; MutableDataParts precommitted_parts; MutableDataParts locked_parts; - bool has_in_memory_parts = false; void clear(); }; @@ -563,10 +567,11 @@ public: Transaction & out_transaction); /// Unlocked version of previous one. Useful when added multiple parts with a single lock. - DataPartsVector renameTempPartAndReplaceUnlocked( + bool renameTempPartAndReplaceUnlocked( MutableDataPartPtr & part, Transaction & out_transaction, - DataPartsLock & lock); + DataPartsLock & lock, + DataPartsVector * out_covered_parts = nullptr); /// Remove parts from working set immediately (without wait for background /// process). Transfer part state to temporary. Have very limited usage only @@ -796,6 +801,9 @@ public: std::unordered_set getPartitionIDsFromQuery(const ASTs & asts, ContextPtr context) const; std::set getPartitionIdsAffectedByCommands(const MutationCommands & commands, ContextPtr query_context) const; + /// Returns set of partition_ids of all Active parts + std::unordered_set getAllPartitionIds() const; + /// Extracts MergeTreeData of other *MergeTree* storage /// and checks that their structure suitable for ALTER TABLE ATTACH PARTITION FROM /// Tables structure should be locked. @@ -917,6 +925,9 @@ public: using WriteAheadLogPtr = std::shared_ptr; WriteAheadLogPtr getWriteAheadLog(); + constexpr static auto EMPTY_PART_TMP_PREFIX = "tmp_empty_"; + MergeTreeData::MutableDataPartPtr createEmptyPart(MergeTreePartInfo & new_part_info, const MergeTreePartition & partition, const String & new_part_name, const MergeTreeTransactionPtr & txn); + MergeTreeDataFormatVersion format_version; /// Merging params - what additional actions to perform during merge. @@ -1025,7 +1036,7 @@ public: using MatcherFn = std::function; /// Returns an object that protects temporary directory from cleanup - scope_guard getTemporaryPartDirectoryHolder(const String & part_dir_name); + scope_guard getTemporaryPartDirectoryHolder(const String & part_dir_name) const; protected: friend class IMergeTreeDataPart; @@ -1108,6 +1119,10 @@ protected: DataPartsIndexes::index::type & data_parts_by_info; DataPartsIndexes::index::type & data_parts_by_state_and_info; + /// Mutex for critical sections which alter set of parts + /// It is like truncate, drop/detach partition + mutable std::mutex operation_with_data_parts_mutex; + /// Current description of columns of data type Object. /// It changes only when set of parts is changed and is /// protected by @data_parts_mutex. @@ -1217,6 +1232,23 @@ protected: DataPartPtr & out_covering_part, DataPartsLock & data_parts_lock) const; + DataPartsVector getCoveredOutdatedParts( + const DataPartPtr & part, + DataPartsLock & data_parts_lock) const; + + struct PartHierarchy + { + DataPartPtr duplicate_part; + DataPartsVector covering_parts; + DataPartsVector covered_parts; + DataPartsVector intersected_parts; + }; + + PartHierarchy getPartHierarchy( + const MergeTreePartInfo & part_info, + DataPartState state, + DataPartsLock & /* data_parts_lock */) const; + /// Checks whether the column is in the primary key, possibly wrapped in a chain of functions with single argument. bool isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const ASTPtr & node, const StorageMetadataPtr & metadata_snapshot) const; @@ -1286,8 +1318,9 @@ protected: static void incrementMergedPartsProfileEvent(MergeTreeDataPartType type); private: - /// Checking that candidate part doesn't break invariants: correct partition and doesn't exist already - void checkPartCanBeAddedToTable(MutableDataPartPtr & part, DataPartsLock & lock) const; + /// Checking that candidate part doesn't break invariants: correct partition + void checkPartPartition(MutableDataPartPtr & part, DataPartsLock & lock) const; + void checkPartDuplicate(MutableDataPartPtr & part, Transaction & transaction, DataPartsLock & lock) const; /// Preparing itself to be committed in memory: fill some fields inside part, add it to data_parts_indexes /// in precommitted state and to transaction @@ -1377,7 +1410,7 @@ private: static MutableDataPartPtr preparePartForRemoval(const DataPartPtr & part); - TemporaryParts temporary_parts; + mutable TemporaryParts temporary_parts; }; /// RAII struct to record big parts that are submerging or emerging. diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 6e3577159cf..79670c0ab27 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -244,7 +244,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge( * So we have to check if this part is currently being inserted with quorum and so on and so forth. * Obviously we have to check it manually only for the first part * of each partition because it will be automatically checked for a pair of parts. */ - if (!can_merge_callback(nullptr, part, txn.get(), nullptr)) + if (!can_merge_callback(nullptr, part, txn.get(), out_disable_reason)) continue; /// This part can be merged only with next parts (no prev part exists), so start @@ -256,7 +256,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge( { /// If we cannot merge with previous part we had to start new parts /// interval (in the same partition) - if (!can_merge_callback(*prev_part, part, txn.get(), nullptr)) + if (!can_merge_callback(*prev_part, part, txn.get(), out_disable_reason)) { /// Now we have no previous part prev_part = nullptr; @@ -268,7 +268,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge( /// for example, merge is already assigned for such parts, or they participate in quorum inserts /// and so on. /// Also we don't start new interval here (maybe all next parts cannot be merged and we don't want to have empty interval) - if (!can_merge_callback(nullptr, part, txn.get(), nullptr)) + if (!can_merge_callback(nullptr, part, txn.get(), out_disable_reason)) continue; /// Starting new interval in the same partition diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index d085bb29b20..a887b0ee322 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -244,7 +244,7 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex(const Bloc const auto & primary_column = primary_index_block.getByPosition(j); index_columns[j]->insertFrom(*primary_column.column, granule.start_row); primary_column.type->getDefaultSerialization()->serializeBinary( - *primary_column.column, granule.start_row, compress_primary_key ? *index_source_hashing_stream : *index_file_hashing_stream); + *primary_column.column, granule.start_row, compress_primary_key ? *index_source_hashing_stream : *index_file_hashing_stream, {}); } } } @@ -312,7 +312,7 @@ void MergeTreeDataPartWriterOnDisk::fillPrimaryIndexChecksums(MergeTreeData::Dat size_t last_row_number = column.size() - 1; index_columns[j]->insertFrom(column, last_row_number); index_types[j]->getDefaultSerialization()->serializeBinary( - column, last_row_number, compress_primary_key ? *index_source_hashing_stream : *index_file_hashing_stream); + column, last_row_number, compress_primary_key ? *index_source_hashing_stream : *index_file_hashing_stream, {}); } last_block_index_columns.clear(); } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 0318fc0648c..8f824ca0777 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -408,7 +408,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( /* group_by_use_nulls */ false, std::move(group_by_info), std::move(group_by_sort_description), - should_produce_results_in_order_of_bucket_number); + should_produce_results_in_order_of_bucket_number, + settings.enable_memory_bound_merging_of_aggregation_results); query_plan->addStep(std::move(aggregating_step)); }; @@ -1113,6 +1114,10 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd for (size_t part_index = 0; part_index < parts.size(); ++part_index) pool.scheduleOrThrowOnError([&, part_index, thread_group = CurrentThread::getGroup()] { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); if (thread_group) CurrentThread::attachToIfDetached(thread_group); diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index d5d2c68b190..2020796f925 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -27,6 +27,8 @@ struct MergeTreeReaderSettings bool read_in_order = false; /// Deleted mask is applied to all reads except internal select from mutate some part columns. bool apply_deleted_mask = true; + /// Put reading task in a common I/O pool, return Async state on prepare() + bool use_asynchronous_read_from_pool = false; }; struct MergeTreeWriterSettings diff --git a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp index 655ca003deb..0882b7fa129 100644 --- a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp @@ -8,7 +8,7 @@ namespace ErrorCodes extern const int MEMORY_LIMIT_EXCEEDED; } -bool MergeTreeInOrderSelectProcessor::getNewTaskImpl() +bool MergeTreeInOrderSelectAlgorithm::getNewTaskImpl() try { if (all_mark_ranges.empty()) diff --git a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h index feacc159d7e..f7c3f294658 100644 --- a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h @@ -8,12 +8,12 @@ namespace DB /// Used to read data from single part with select query in order of primary key. /// Cares about PREWHERE, virtual columns, indexes etc. /// To read data from multiple parts, Storage (MergeTree) creates multiple such objects. -class MergeTreeInOrderSelectProcessor final : public MergeTreeSelectProcessor +class MergeTreeInOrderSelectAlgorithm final : public MergeTreeSelectAlgorithm { public: template - explicit MergeTreeInOrderSelectProcessor(Args &&... args) - : MergeTreeSelectProcessor{std::forward(args)...} + explicit MergeTreeInOrderSelectAlgorithm(Args &&... args) + : MergeTreeSelectAlgorithm{std::forward(args)...} { LOG_TRACE(log, "Reading {} ranges in order from part {}, approx. {} rows starting from {}", all_mark_ranges.size(), data_part->name, total_rows, diff --git a/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp b/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp index 743bb504dbd..4dd0614015c 100644 --- a/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp @@ -70,15 +70,18 @@ namespace ErrorCodes extern const int INCORRECT_NUMBER_OF_COLUMNS; extern const int INCORRECT_QUERY; extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; } -MergeTreeIndexGranuleAnnoy::MergeTreeIndexGranuleAnnoy(const String & index_name_, const Block & index_sample_block_) +template +MergeTreeIndexGranuleAnnoy::MergeTreeIndexGranuleAnnoy(const String & index_name_, const Block & index_sample_block_) : index_name(index_name_) , index_sample_block(index_sample_block_) , index(nullptr) {} -MergeTreeIndexGranuleAnnoy::MergeTreeIndexGranuleAnnoy( +template +MergeTreeIndexGranuleAnnoy::MergeTreeIndexGranuleAnnoy( const String & index_name_, const Block & index_sample_block_, AnnoyIndexPtr index_base_) @@ -87,7 +90,8 @@ MergeTreeIndexGranuleAnnoy::MergeTreeIndexGranuleAnnoy( , index(std::move(index_base_)) {} -void MergeTreeIndexGranuleAnnoy::serializeBinary(WriteBuffer & ostr) const +template +void MergeTreeIndexGranuleAnnoy::serializeBinary(WriteBuffer & ostr) const { /// number of dimensions is required in the constructor, /// so it must be written and read separately from the other part @@ -95,7 +99,8 @@ void MergeTreeIndexGranuleAnnoy::serializeBinary(WriteBuffer & ostr) const index->serialize(ostr); } -void MergeTreeIndexGranuleAnnoy::deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion /*version*/) +template +void MergeTreeIndexGranuleAnnoy::deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion /*version*/) { uint64_t dimension; readIntBinary(dimension, istr); @@ -103,8 +108,8 @@ void MergeTreeIndexGranuleAnnoy::deserializeBinary(ReadBuffer & istr, MergeTreeI index->deserialize(istr); } - -MergeTreeIndexAggregatorAnnoy::MergeTreeIndexAggregatorAnnoy( +template +MergeTreeIndexAggregatorAnnoy::MergeTreeIndexAggregatorAnnoy( const String & index_name_, const Block & index_sample_block_, uint64_t number_of_trees_) @@ -113,16 +118,18 @@ MergeTreeIndexAggregatorAnnoy::MergeTreeIndexAggregatorAnnoy( , number_of_trees(number_of_trees_) {} -MergeTreeIndexGranulePtr MergeTreeIndexAggregatorAnnoy::getGranuleAndReset() +template +MergeTreeIndexGranulePtr MergeTreeIndexAggregatorAnnoy::getGranuleAndReset() { // NOLINTNEXTLINE(*) index->build(static_cast(number_of_trees), /*number_of_threads=*/1); - auto granule = std::make_shared(index_name, index_sample_block, index); + auto granule = std::make_shared >(index_name, index_sample_block, index); index = nullptr; return granule; } -void MergeTreeIndexAggregatorAnnoy::update(const Block & block, size_t * pos, size_t limit) +template +void MergeTreeIndexAggregatorAnnoy::update(const Block & block, size_t * pos, size_t limit) { if (*pos >= block.rows()) throw Exception( @@ -193,8 +200,9 @@ void MergeTreeIndexAggregatorAnnoy::update(const Block & block, size_t * pos, si MergeTreeIndexConditionAnnoy::MergeTreeIndexConditionAnnoy( const IndexDescription & /*index*/, const SelectQueryInfo & query, - ContextPtr context) - : condition(query, context) + ContextPtr context, + const String& distance_name_) + : condition(query, context), distance_name(distance_name_) {} @@ -205,10 +213,28 @@ bool MergeTreeIndexConditionAnnoy::mayBeTrueOnGranule(MergeTreeIndexGranulePtr / bool MergeTreeIndexConditionAnnoy::alwaysUnknownOrTrue() const { - return condition.alwaysUnknownOrTrue("L2Distance"); + return condition.alwaysUnknownOrTrue(distance_name); } std::vector MergeTreeIndexConditionAnnoy::getUsefulRanges(MergeTreeIndexGranulePtr idx_granule) const +{ + if (distance_name == "L2Distance") + { + return getUsefulRangesImpl<::Annoy::Euclidean>(idx_granule); + } + else if (distance_name == "cosineDistance") + { + return getUsefulRangesImpl<::Annoy::Angular>(idx_granule); + } + else + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown distance name. Must be 'L2Distance' or 'cosineDistance'. Got {}", distance_name); + } +} + + +template +std::vector MergeTreeIndexConditionAnnoy::getUsefulRangesImpl(MergeTreeIndexGranulePtr idx_granule) const { UInt64 limit = condition.getLimit(); UInt64 index_granularity = condition.getIndexGranularity(); @@ -220,7 +246,7 @@ std::vector MergeTreeIndexConditionAnnoy::getUsefulRanges(MergeTreeIndex std::vector target_vec = condition.getTargetVector(); - auto granule = std::dynamic_pointer_cast(idx_granule); + auto granule = std::dynamic_pointer_cast >(idx_granule); if (granule == nullptr) throw Exception("Granule has the wrong type", ErrorCodes::LOGICAL_ERROR); @@ -267,33 +293,54 @@ std::vector MergeTreeIndexConditionAnnoy::getUsefulRanges(MergeTreeIndex return result_vector; } - -MergeTreeIndexAnnoy::MergeTreeIndexAnnoy(const IndexDescription & index_, uint64_t number_of_trees_) - : IMergeTreeIndex(index_) - , number_of_trees(number_of_trees_) -{ -} - MergeTreeIndexGranulePtr MergeTreeIndexAnnoy::createIndexGranule() const { - return std::make_shared(index.name, index.sample_block); + if (distance_name == "L2Distance") + { + return std::make_shared >(index.name, index.sample_block); + } + if (distance_name == "cosineDistance") + { + return std::make_shared >(index.name, index.sample_block); + } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown distance name. Must be 'L2Distance' or 'cosineDistance'. Got {}", distance_name); } MergeTreeIndexAggregatorPtr MergeTreeIndexAnnoy::createIndexAggregator() const { - return std::make_shared(index.name, index.sample_block, number_of_trees); + if (distance_name == "L2Distance") + { + return std::make_shared >(index.name, index.sample_block, number_of_trees); + } + if (distance_name == "cosineDistance") + { + return std::make_shared >(index.name, index.sample_block, number_of_trees); + } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown distance name. Must be 'L2Distance' or 'cosineDistance'. Got {}", distance_name); } MergeTreeIndexConditionPtr MergeTreeIndexAnnoy::createIndexCondition( const SelectQueryInfo & query, ContextPtr context) const { - return std::make_shared(index, query, context); + return std::make_shared(index, query, context, distance_name); }; MergeTreeIndexPtr annoyIndexCreator(const IndexDescription & index) { - uint64_t param = index.arguments[0].get(); - return std::make_shared(index, param); + uint64_t param = 100; + String distance_name = "L2Distance"; + if (!index.arguments.empty() && !index.arguments[0].tryGet(param)) + { + if (!index.arguments[0].tryGet(distance_name)) + { + throw Exception("Can't parse first argument", ErrorCodes::INCORRECT_DATA); + } + } + if (index.arguments.size() > 1 && !index.arguments[1].tryGet(distance_name)) + { + throw Exception("Can't parse second argument", ErrorCodes::INCORRECT_DATA); + } + return std::make_shared(index, param, distance_name); } static void assertIndexColumnsType(const Block & header) @@ -332,13 +379,18 @@ static void assertIndexColumnsType(const Block & header) void annoyIndexValidator(const IndexDescription & index, bool /* attach */) { - if (index.arguments.size() != 1) + if (index.arguments.size() > 2) { - throw Exception("Annoy index must have exactly one argument.", ErrorCodes::INCORRECT_QUERY); + throw Exception("Annoy index must not have more than two parameters", ErrorCodes::INCORRECT_QUERY); } - if (index.arguments[0].getType() != Field::Types::UInt64) + if (!index.arguments.empty() && index.arguments[0].getType() != Field::Types::UInt64 + && index.arguments[0].getType() != Field::Types::String) { - throw Exception("Annoy index argument must be UInt64.", ErrorCodes::INCORRECT_QUERY); + throw Exception("Annoy index first argument must be UInt64 or String.", ErrorCodes::INCORRECT_QUERY); + } + if (index.arguments.size() > 1 && index.arguments[1].getType() != Field::Types::String) + { + throw Exception("Annoy index second argument must be String.", ErrorCodes::INCORRECT_QUERY); } if (index.column_names.size() != 1 || index.data_types.size() != 1) diff --git a/src/Storages/MergeTree/MergeTreeIndexAnnoy.h b/src/Storages/MergeTree/MergeTreeIndexAnnoy.h index 6a844947bd2..3b1a41eb85d 100644 --- a/src/Storages/MergeTree/MergeTreeIndexAnnoy.h +++ b/src/Storages/MergeTree/MergeTreeIndexAnnoy.h @@ -17,7 +17,7 @@ namespace ApproximateNearestNeighbour using AnnoyIndexThreadedBuildPolicy = ::Annoy::AnnoyIndexMultiThreadedBuildPolicy; // TODO: Support different metrics. List of available metrics can be taken from here: // https://github.com/spotify/annoy/blob/master/src/annoymodule.cc#L151-L171 - template + template class AnnoyIndex : public ::Annoy::AnnoyIndex { using Base = ::Annoy::AnnoyIndex; @@ -29,9 +29,10 @@ namespace ApproximateNearestNeighbour }; } +template struct MergeTreeIndexGranuleAnnoy final : public IMergeTreeIndexGranule { - using AnnoyIndex = ApproximateNearestNeighbour::AnnoyIndex<>; + using AnnoyIndex = ApproximateNearestNeighbour::AnnoyIndex; using AnnoyIndexPtr = std::shared_ptr; MergeTreeIndexGranuleAnnoy(const String & index_name_, const Block & index_sample_block_); @@ -52,10 +53,10 @@ struct MergeTreeIndexGranuleAnnoy final : public IMergeTreeIndexGranule AnnoyIndexPtr index; }; - +template struct MergeTreeIndexAggregatorAnnoy final : IMergeTreeIndexAggregator { - using AnnoyIndex = ApproximateNearestNeighbour::AnnoyIndex<>; + using AnnoyIndex = ApproximateNearestNeighbour::AnnoyIndex; using AnnoyIndexPtr = std::shared_ptr; MergeTreeIndexAggregatorAnnoy(const String & index_name_, const Block & index_sample_block, uint64_t number_of_trees); @@ -78,7 +79,8 @@ public: MergeTreeIndexConditionAnnoy( const IndexDescription & index, const SelectQueryInfo & query, - ContextPtr context); + ContextPtr context, + const String& distance_name); bool alwaysUnknownOrTrue() const override; @@ -89,14 +91,24 @@ public: ~MergeTreeIndexConditionAnnoy() override = default; private: + template + std::vector getUsefulRangesImpl(MergeTreeIndexGranulePtr idx_granule) const; + ApproximateNearestNeighbour::ANNCondition condition; + const String distance_name; }; class MergeTreeIndexAnnoy : public IMergeTreeIndex { public: - MergeTreeIndexAnnoy(const IndexDescription & index_, uint64_t number_of_trees_); + + MergeTreeIndexAnnoy(const IndexDescription & index_, uint64_t number_of_trees_, const String& distance_name_) + : IMergeTreeIndex(index_) + , number_of_trees(number_of_trees_) + , distance_name(distance_name_) + {} + ~MergeTreeIndexAnnoy() override = default; MergeTreeIndexGranulePtr createIndexGranule() const override; @@ -109,6 +121,7 @@ public: private: const uint64_t number_of_trees; + const String distance_name; }; diff --git a/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp b/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp index 088029d9e8e..d8765ddb9bc 100644 --- a/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp @@ -26,7 +26,7 @@ MergeTreeIndexGranuleHypothesis::MergeTreeIndexGranuleHypothesis(const String & void MergeTreeIndexGranuleHypothesis::serializeBinary(WriteBuffer & ostr) const { const auto & size_type = DataTypePtr(std::make_shared()); - size_type->getDefaultSerialization()->serializeBinary(static_cast(met), ostr); + size_type->getDefaultSerialization()->serializeBinary(static_cast(met), ostr, {}); } void MergeTreeIndexGranuleHypothesis::deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion version) @@ -36,7 +36,7 @@ void MergeTreeIndexGranuleHypothesis::deserializeBinary(ReadBuffer & istr, Merge Field field_met; const auto & size_type = DataTypePtr(std::make_shared()); - size_type->getDefaultSerialization()->deserializeBinary(field_met, istr); + size_type->getDefaultSerialization()->deserializeBinary(field_met, istr, {}); met = field_met.get(); is_empty = false; } diff --git a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp index 43e655a4ee5..fc19f819cf1 100644 --- a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp @@ -42,8 +42,8 @@ void MergeTreeIndexGranuleMinMax::serializeBinary(WriteBuffer & ostr) const const DataTypePtr & type = index_sample_block.getByPosition(i).type; auto serialization = type->getDefaultSerialization(); - serialization->serializeBinary(hyperrectangle[i].left, ostr); - serialization->serializeBinary(hyperrectangle[i].right, ostr); + serialization->serializeBinary(hyperrectangle[i].left, ostr, {}); + serialization->serializeBinary(hyperrectangle[i].right, ostr, {}); } } @@ -63,8 +63,8 @@ void MergeTreeIndexGranuleMinMax::deserializeBinary(ReadBuffer & istr, MergeTree case 1: if (!type->isNullable()) { - serialization->deserializeBinary(min_val, istr); - serialization->deserializeBinary(max_val, istr); + serialization->deserializeBinary(min_val, istr, {}); + serialization->deserializeBinary(max_val, istr, {}); } else { @@ -78,8 +78,8 @@ void MergeTreeIndexGranuleMinMax::deserializeBinary(ReadBuffer & istr, MergeTree readBinary(is_null, istr); if (!is_null) { - serialization->deserializeBinary(min_val, istr); - serialization->deserializeBinary(max_val, istr); + serialization->deserializeBinary(min_val, istr, {}); + serialization->deserializeBinary(max_val, istr, {}); } else { @@ -91,8 +91,8 @@ void MergeTreeIndexGranuleMinMax::deserializeBinary(ReadBuffer & istr, MergeTree /// New format with proper Nullable support for values that includes Null values case 2: - serialization->deserializeBinary(min_val, istr); - serialization->deserializeBinary(max_val, istr); + serialization->deserializeBinary(min_val, istr, {}); + serialization->deserializeBinary(max_val, istr, {}); // NULL_LAST if (min_val.isNull()) diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp index 0e15f2c4cb6..a28394e943e 100644 --- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp @@ -56,11 +56,11 @@ void MergeTreeIndexGranuleSet::serializeBinary(WriteBuffer & ostr) const if (max_rows != 0 && size() > max_rows) { - size_serialization->serializeBinary(0, ostr); + size_serialization->serializeBinary(0, ostr, {}); return; } - size_serialization->serializeBinary(size(), ostr); + size_serialization->serializeBinary(size(), ostr, {}); for (size_t i = 0; i < index_sample_block.columns(); ++i) { @@ -90,7 +90,7 @@ void MergeTreeIndexGranuleSet::deserializeBinary(ReadBuffer & istr, MergeTreeInd Field field_rows; const auto & size_type = DataTypePtr(std::make_shared()); - size_type->getDefaultSerialization()->deserializeBinary(field_rows, istr); + size_type->getDefaultSerialization()->deserializeBinary(field_rows, istr, {}); size_t rows_to_read = field_rows.get(); if (rows_to_read == 0) diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 10f5cc95baf..e7fdf1617f0 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -379,7 +379,7 @@ void MergeTreePartition::load(const MergeTreeData & storage, const PartMetadataM auto file = manager->read("partition.dat"); value.resize(partition_key_sample.columns()); for (size_t i = 0; i < partition_key_sample.columns(); ++i) - partition_key_sample.getByPosition(i).type->getDefaultSerialization()->deserializeBinary(value[i], *file); + partition_key_sample.getByPosition(i).type->getDefaultSerialization()->deserializeBinary(value[i], *file, {}); } std::unique_ptr MergeTreePartition::store(const MergeTreeData & storage, IDataPartStorage & data_part_storage, MergeTreeDataPartChecksums & checksums) const @@ -399,7 +399,7 @@ std::unique_ptr MergeTreePartition::store(const Block & HashingWriteBuffer out_hashing(*out); for (size_t i = 0; i < value.size(); ++i) { - partition_key_sample.getByPosition(i).type->getDefaultSerialization()->serializeBinary(value[i], out_hashing); + partition_key_sample.getByPosition(i).type->getDefaultSerialization()->serializeBinary(value[i], out_hashing, {}); } out_hashing.next(); diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 4681f8229ab..f447ef87d00 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -217,7 +217,7 @@ std::vector MergeTreeReadPool::fillPerPartInfo(const RangesInDataParts & column_names, virtual_column_names, prewhere_info, /*with_subcolumns=*/ true); auto size_predictor = !predict_block_size_bytes ? nullptr - : MergeTreeBaseSelectProcessor::getSizePredictor(part.data_part, task_columns, sample_block); + : IMergeTreeSelectAlgorithm::getSizePredictor(part.data_part, task_columns, sample_block); auto & per_part = per_part_params.emplace_back(); diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index c009e6f1165..d0d464b3c29 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -8,7 +8,7 @@ namespace ErrorCodes extern const int MEMORY_LIMIT_EXCEEDED; } -bool MergeTreeReverseSelectProcessor::getNewTaskImpl() +bool MergeTreeReverseSelectAlgorithm::getNewTaskImpl() try { if (chunks.empty() && all_mark_ranges.empty()) @@ -44,9 +44,9 @@ catch (...) throw; } -MergeTreeBaseSelectProcessor::BlockAndRowCount MergeTreeReverseSelectProcessor::readFromPart() +MergeTreeReverseSelectAlgorithm::BlockAndProgress MergeTreeReverseSelectAlgorithm::readFromPart() { - BlockAndRowCount res; + BlockAndProgress res; if (!chunks.empty()) { diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h index 06a218abafa..ccadb1f1c61 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h @@ -9,12 +9,12 @@ namespace DB /// in reverse order of primary key. /// Cares about PREWHERE, virtual columns, indexes etc. /// To read data from multiple parts, Storage (MergeTree) creates multiple such objects. -class MergeTreeReverseSelectProcessor final : public MergeTreeSelectProcessor +class MergeTreeReverseSelectAlgorithm final : public MergeTreeSelectAlgorithm { public: template - explicit MergeTreeReverseSelectProcessor(Args &&... args) - : MergeTreeSelectProcessor{std::forward(args)...} + explicit MergeTreeReverseSelectAlgorithm(Args &&... args) + : MergeTreeSelectAlgorithm{std::forward(args)...} { LOG_TRACE(log, "Reading {} ranges in reverse order from part {}, approx. {} rows starting from {}", all_mark_ranges.size(), data_part->name, total_rows, @@ -27,9 +27,9 @@ private: bool getNewTaskImpl() override; void finalizeNewTask() override {} - BlockAndRowCount readFromPart() override; + BlockAndProgress readFromPart() override; - std::vector chunks; + std::vector chunks; Poco::Logger * log = &Poco::Logger::get("MergeTreeReverseSelectProcessor"); }; diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 2490eb77772..3f9da9c130a 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -8,7 +8,7 @@ namespace DB { -MergeTreeSelectProcessor::MergeTreeSelectProcessor( +MergeTreeSelectAlgorithm::MergeTreeSelectAlgorithm( const MergeTreeData & storage_, const StorageSnapshotPtr & storage_snapshot_, const MergeTreeData::DataPartPtr & owned_data_part_, @@ -25,7 +25,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( size_t part_index_in_query_, bool has_limit_below_one_block_, std::optional extension_) - : MergeTreeBaseSelectProcessor{ + : IMergeTreeSelectAlgorithm{ storage_snapshot_->getSampleBlockForColumns(required_columns_), storage_, storage_snapshot_, prewhere_info_, std::move(actions_settings), max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, @@ -38,10 +38,10 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( has_limit_below_one_block(has_limit_below_one_block_), total_rows(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges)) { - ordered_names = header_without_virtual_columns.getNames(); + ordered_names = header_without_const_virtual_columns.getNames(); } -void MergeTreeSelectProcessor::initializeReaders() +void MergeTreeSelectAlgorithm::initializeReaders() { task_columns = getReadTaskColumns( LoadedMergeTreeDataPartInfoForReader(data_part), storage_snapshot, @@ -61,7 +61,7 @@ void MergeTreeSelectProcessor::initializeReaders() } -void MergeTreeSelectProcessor::finish() +void MergeTreeSelectAlgorithm::finish() { /** Close the files (before destroying the object). * When many sources are created, but simultaneously reading only a few of them, @@ -72,6 +72,6 @@ void MergeTreeSelectProcessor::finish() data_part.reset(); } -MergeTreeSelectProcessor::~MergeTreeSelectProcessor() = default; +MergeTreeSelectAlgorithm::~MergeTreeSelectAlgorithm() = default; } diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 4b3a46fc53c..12f4804835c 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -13,10 +13,10 @@ namespace DB /// Used to read data from single part with select query /// Cares about PREWHERE, virtual columns, indexes etc. /// To read data from multiple parts, Storage (MergeTree) creates multiple such objects. -class MergeTreeSelectProcessor : public MergeTreeBaseSelectProcessor +class MergeTreeSelectAlgorithm : public IMergeTreeSelectAlgorithm { public: - MergeTreeSelectProcessor( + MergeTreeSelectAlgorithm( const MergeTreeData & storage, const StorageSnapshotPtr & storage_snapshot_, const MergeTreeData::DataPartPtr & owned_data_part, @@ -34,13 +34,13 @@ public: bool has_limit_below_one_block_ = false, std::optional extension_ = {}); - ~MergeTreeSelectProcessor() override; + ~MergeTreeSelectAlgorithm() override; protected: /// Defer initialization from constructor, because it may be heavy /// and it's better to do it lazily in `getNewTaskImpl`, which is executing in parallel. void initializeReaders(); - void finish() override final; + void finish() final; /// Used by Task Names required_columns; diff --git a/src/Storages/MergeTree/MergeTreeSource.cpp b/src/Storages/MergeTree/MergeTreeSource.cpp new file mode 100644 index 00000000000..6c9c0508bda --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeSource.cpp @@ -0,0 +1,224 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +MergeTreeSource::MergeTreeSource(MergeTreeSelectAlgorithmPtr algorithm_) + : ISource(algorithm_->getHeader()) + , algorithm(std::move(algorithm_)) +{ +#if defined(OS_LINUX) + if (algorithm->getSettings().use_asynchronous_read_from_pool) + async_reading_state = std::make_unique(); +#endif +} + +MergeTreeSource::~MergeTreeSource() = default; + +std::string MergeTreeSource::getName() const +{ + return algorithm->getName(); +} + +void MergeTreeSource::onCancel() +{ + algorithm->cancel(); +} + +#if defined(OS_LINUX) +struct MergeTreeSource::AsyncReadingState +{ + /// NotStarted -> InProgress -> IsFinished -> NotStarted ... + enum class Stage + { + NotStarted, + InProgress, + IsFinished, + }; + + struct Control + { + /// setResult and setException are the only methods + /// which can be called from background thread. + /// Invariant: + /// * background thread changes status InProgress -> IsFinished + /// * (status == InProgress) => (MergeTreeBaseSelectProcessor is alive) + + void setResult(ChunkAndProgress chunk_) + { + chassert(stage == Stage::InProgress); + chunk = std::move(chunk_); + finish(); + } + + void setException(std::exception_ptr exception_) + { + chassert(stage == Stage::InProgress); + exception = exception_; + finish(); + } + + private: + + /// Executor requires file descriptor (which can be polled) to be returned for async execution. + /// We are using EventFD here. + /// Thread from background pool writes to fd when task is finished. + /// Working thread should read from fd when task is finished or canceled to wait for bg thread. + EventFD event; + std::atomic stage = Stage::NotStarted; + + ChunkAndProgress chunk; + std::exception_ptr exception; + + void finish() + { + stage = Stage::IsFinished; + event.write(); + } + + ChunkAndProgress getResult() + { + chassert(stage == Stage::IsFinished); + event.read(); + stage = Stage::NotStarted; + + if (exception) + std::rethrow_exception(exception); + + return std::move(chunk); + } + + friend struct AsyncReadingState; + }; + + std::shared_ptr start() + { + chassert(control->stage == Stage::NotStarted); + control->stage = Stage::InProgress; + return control; + } + + void schedule(ThreadPool::Job job) + { + callback_runner(std::move(job), 0); + } + + ChunkAndProgress getResult() + { + return control->getResult(); + } + + Stage getStage() const { return control->stage; } + int getFD() const { return control->event.fd; } + + AsyncReadingState() + { + control = std::make_shared(); + callback_runner = threadPoolCallbackRunner(IOThreadPool::get(), "MergeTreeRead"); + } + + ~AsyncReadingState() + { + /// Here we wait for async task if needed. + /// ~AsyncReadingState and Control::finish can be run concurrently. + /// It's important to store std::shared_ptr into bg pool task. + /// Otherwise following is possible: + /// + /// (executing thread) (bg pool thread) + /// Control::finish() + /// stage = Stage::IsFinished; + /// ~MergeTreeBaseSelectProcessor() + /// ~AsyncReadingState() + /// control->stage != Stage::InProgress + /// ~EventFD() + /// event.write() + if (control->stage == Stage::InProgress) + control->event.read(); + } + +private: + ThreadPoolCallbackRunner callback_runner; + std::shared_ptr control; +}; +#endif + +ISource::Status MergeTreeSource::prepare() +{ +#if defined(OS_LINUX) + if (!async_reading_state) + return ISource::prepare(); + + /// Check if query was cancelled before returning Async status. Otherwise it may lead to infinite loop. + if (isCancelled()) + { + getPort().finish(); + return ISource::Status::Finished; + } + + if (async_reading_state && async_reading_state->getStage() == AsyncReadingState::Stage::InProgress) + return ISource::Status::Async; +#endif + + return ISource::prepare(); +} + + +std::optional MergeTreeSource::reportProgress(ChunkAndProgress chunk) +{ + if (chunk.num_read_rows || chunk.num_read_bytes) + progress(chunk.num_read_rows, chunk.num_read_bytes); + + if (chunk.chunk.hasRows()) + return std::move(chunk.chunk); + + return {}; +} + + +std::optional MergeTreeSource::tryGenerate() +{ +#if defined(OS_LINUX) + if (async_reading_state) + { + if (async_reading_state->getStage() == AsyncReadingState::Stage::IsFinished) + return reportProgress(async_reading_state->getResult()); + + chassert(async_reading_state->getStage() == AsyncReadingState::Stage::NotStarted); + + /// It is important to store control into job. + /// Otherwise, race between job and ~MergeTreeBaseSelectProcessor is possible. + auto job = [this, control = async_reading_state->start()]() mutable + { + auto holder = std::move(control); + + try + { + holder->setResult(algorithm->read()); + } + catch (...) + { + holder->setException(std::current_exception()); + } + }; + + async_reading_state->schedule(std::move(job)); + + return Chunk(); + } +#endif + + return reportProgress(algorithm->read()); +} + +#if defined(OS_LINUX) +int MergeTreeSource::schedule() +{ + return async_reading_state->getFD(); +} +#endif + +} diff --git a/src/Storages/MergeTree/MergeTreeSource.h b/src/Storages/MergeTree/MergeTreeSource.h new file mode 100644 index 00000000000..bba0c0af80e --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeSource.h @@ -0,0 +1,42 @@ +#pragma once +#include + +namespace DB +{ + +class IMergeTreeSelectAlgorithm; +using MergeTreeSelectAlgorithmPtr = std::unique_ptr; + +struct ChunkAndProgress; + +class MergeTreeSource final : public ISource +{ +public: + explicit MergeTreeSource(MergeTreeSelectAlgorithmPtr algorithm_); + ~MergeTreeSource() override; + + std::string getName() const override; + + Status prepare() override; + +#if defined(OS_LINUX) + int schedule() override; +#endif + +protected: + std::optional tryGenerate() override; + + void onCancel() override; + +private: + MergeTreeSelectAlgorithmPtr algorithm; + +#if defined(OS_LINUX) + struct AsyncReadingState; + std::unique_ptr async_reading_state; +#endif + + std::optional reportProgress(ChunkAndProgress chunk); +}; + +} diff --git a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp index 04b7f6094e4..60586024359 100644 --- a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp @@ -12,7 +12,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -MergeTreeThreadSelectProcessor::MergeTreeThreadSelectProcessor( +MergeTreeThreadSelectAlgorithm::MergeTreeThreadSelectAlgorithm( size_t thread_, const MergeTreeReadPoolPtr & pool_, size_t min_marks_to_read_, @@ -28,7 +28,7 @@ MergeTreeThreadSelectProcessor::MergeTreeThreadSelectProcessor( const Names & virt_column_names_, std::optional extension_) : - MergeTreeBaseSelectProcessor{ + IMergeTreeSelectAlgorithm{ pool_->getHeader(), storage_, storage_snapshot_, prewhere_info_, std::move(actions_settings), max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, reader_settings_, use_uncompressed_cache_, virt_column_names_, extension_}, @@ -86,18 +86,18 @@ MergeTreeThreadSelectProcessor::MergeTreeThreadSelectProcessor( } - ordered_names = getPort().getHeader().getNames(); + ordered_names = getHeader().getNames(); } /// Requests read task from MergeTreeReadPool and signals whether it got one -bool MergeTreeThreadSelectProcessor::getNewTaskImpl() +bool MergeTreeThreadSelectAlgorithm::getNewTaskImpl() { task = pool->getTask(min_marks_to_read, thread, ordered_names); return static_cast(task); } -void MergeTreeThreadSelectProcessor::finalizeNewTask() +void MergeTreeThreadSelectAlgorithm::finalizeNewTask() { const std::string part_name = task->data_part->isProjectionPart() ? task->data_part->getParentPart()->name : task->data_part->name; @@ -129,13 +129,13 @@ void MergeTreeThreadSelectProcessor::finalizeNewTask() } -void MergeTreeThreadSelectProcessor::finish() +void MergeTreeThreadSelectAlgorithm::finish() { reader.reset(); pre_reader_for_step.clear(); } -MergeTreeThreadSelectProcessor::~MergeTreeThreadSelectProcessor() = default; +MergeTreeThreadSelectAlgorithm::~MergeTreeThreadSelectAlgorithm() = default; } diff --git a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h index 3bba42bed28..a9104f25d75 100644 --- a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h @@ -11,10 +11,10 @@ class MergeTreeReadPool; /** Used in conjunction with MergeTreeReadPool, asking it for more work to do and performing whatever reads it is asked * to perform. */ -class MergeTreeThreadSelectProcessor final : public MergeTreeBaseSelectProcessor +class MergeTreeThreadSelectAlgorithm final : public IMergeTreeSelectAlgorithm { public: - MergeTreeThreadSelectProcessor( + MergeTreeThreadSelectAlgorithm( size_t thread_, const std::shared_ptr & pool_, size_t min_marks_to_read_, @@ -32,7 +32,7 @@ public: String getName() const override { return "MergeTreeThread"; } - ~MergeTreeThreadSelectProcessor() override; + ~MergeTreeThreadSelectAlgorithm() override; protected: /// Requests read task from MergeTreeReadPool and signals whether it got one diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.h b/src/Storages/MergeTree/MutateFromLogEntryTask.h index 416b0c92522..23c9428faa9 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.h +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.h @@ -20,7 +20,7 @@ public: StorageReplicatedMergeTree & storage_, Callback && task_result_callback_) : ReplicatedMergeMutateTaskBase( - &Poco::Logger::get(storage_.getStorageID().getShortName() + "::" + selected_entry_->log_entry->new_part_name + "(MutateFromLogEntryTask)"), + &Poco::Logger::get(storage_.getStorageID().getShortName() + "::" + selected_entry_->log_entry->new_part_name + " (MutateFromLogEntryTask)"), storage_, selected_entry_, task_result_callback_) diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index e5ba771a198..f6befe67fd4 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -1322,9 +1322,11 @@ private: for (auto p_it = projection_data_part_storage_src->iterate(); p_it->isValid(); p_it->next()) { + auto file_name_with_projection_prefix = fs::path(projection_data_part_storage_src->getPartDirectory()) / p_it->name(); projection_data_part_storage_dst->createHardLinkFrom( *projection_data_part_storage_src, p_it->name(), p_it->name()); - hardlinked_files.insert(p_it->name()); + + hardlinked_files.insert(file_name_with_projection_prefix); } } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 2d7afeafd0d..1f3aac57969 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -732,27 +732,126 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper namespace { -Names getPartNamesToMutate( - const ReplicatedMergeTreeMutationEntry & mutation, const ActiveDataPartSet & parts, const DropPartsRanges & drop_ranges) -{ - Names result; - for (const auto & pair : mutation.block_numbers) - { - const String & partition_id = pair.first; - Int64 block_num = pair.second; +/// Simplified representation of queue entry. Contain two sets +/// 1) Which parts we will receive after entry execution +/// 2) Which parts we will drop/remove after entry execution +/// +/// We use this representation to understand which parts mutation actually have to mutate. +struct QueueEntryRepresentation +{ + std::vector produced_parts; + std::vector dropped_parts; +}; + +using QueueRepresentation = std::map; + +/// Produce a map from queue znode name to simplified entry representation. +QueueRepresentation getQueueRepresentation(const std::list & entries, MergeTreeDataFormatVersion format_version) +{ + using LogEntryType = ReplicatedMergeTreeLogEntryData::Type; + QueueRepresentation result; + for (const auto & entry : entries) + { + const auto & key = entry->znode_name; + switch (entry->type) + { + /// explicetely specify all types of entries without default, so if + /// someone decide to add new type it will produce a compiler warning (error in our case) + case LogEntryType::GET_PART: + case LogEntryType::ATTACH_PART: + case LogEntryType::MERGE_PARTS: + case LogEntryType::MUTATE_PART: + { + result[key].produced_parts.push_back(entry->new_part_name); + break; + } + case LogEntryType::REPLACE_RANGE: + { + /// Quite tricky entry, it both produce and drop parts (in some cases) + const auto & new_parts = entry->replace_range_entry->new_part_names; + auto & produced_parts = result[key].produced_parts; + produced_parts.insert( + produced_parts.end(), new_parts.begin(), new_parts.end()); + + if (auto drop_range = entry->getDropRange(format_version)) + { + auto & dropped_parts = result[key].dropped_parts; + dropped_parts.push_back(*drop_range); + } + break; + } + case LogEntryType::DROP_RANGE: + { + result[key].dropped_parts.push_back(entry->new_part_name); + break; + } + /// These entries don't produce/drop any parts + case LogEntryType::EMPTY: + case LogEntryType::ALTER_METADATA: + case LogEntryType::CLEAR_INDEX: + case LogEntryType::CLEAR_COLUMN: + case LogEntryType::SYNC_PINNED_PART_UUIDS: + case LogEntryType::CLONE_PART_FROM_SHARD: + { + break; + } + } + } + return result; +} + +/// Try to understand which part we need to mutate to finish mutation. In ReplicatedQueue we have two sets of parts: +/// current parts -- set of parts which we actually have (on disk) +/// virtual parts -- set of parts which we will have after we will execute our queue +/// +/// From the first glance it can sound that these two sets should be enough to understand which parts we have to mutate +/// to finish mutation but it's not true: +/// 1) Obviously we cannot rely on current_parts because we can have stale state (some parts are absent, some merges not finished). We also have to account parts which we will +/// get after queue execution. +/// 2) But we cannot rely on virtual_parts for this, because they contain parts which we will get after we have executed our queue. So if we need to execute mutation 0000000001 for part all_0_0_0 +/// and we have already pulled entry to mutate this part into own queue our virtual parts will contain part all_0_0_0_1, not part all_0_0_0. +/// +/// To avoid such issues we simply traverse all entries in queue in order and applying diff (add parts/remove parts) to current parts if they could be affected by mutation. Such approach is expensive +/// but we do it only once since we get the mutation. After that we just update parts_to_do for each mutation when pulling entries into our queue (addPartToMutations, removePartFromMutations). +ActiveDataPartSet getPartNamesToMutate( + const ReplicatedMergeTreeMutationEntry & mutation, const ActiveDataPartSet & current_parts, + const QueueRepresentation & queue_representation, MergeTreeDataFormatVersion format_version) +{ + ActiveDataPartSet result(format_version); + /// Traverse mutation by partition + for (const auto & [partition_id, block_num] : mutation.block_numbers) + { /// Note that we cannot simply count all parts to mutate using getPartsCoveredBy(appropriate part_info) /// because they are not consecutive in `parts`. MergeTreePartInfo covering_part_info( partition_id, 0, block_num, MergeTreePartInfo::MAX_LEVEL, MergeTreePartInfo::MAX_BLOCK_NUMBER); - for (const String & covered_part_name : parts.getPartsCoveredBy(covering_part_info)) + + /// First of all add all affected current_parts + for (const String & covered_part_name : current_parts.getPartsCoveredBy(covering_part_info)) { - auto part_info = MergeTreePartInfo::fromPartName(covered_part_name, parts.getFormatVersion()); + auto part_info = MergeTreePartInfo::fromPartName(covered_part_name, current_parts.getFormatVersion()); if (part_info.getDataVersion() < block_num) + result.add(covered_part_name); + } + + /// Traverse queue and update affected current_parts + for (const auto & [_, entry_representation] : queue_representation) + { + /// First we have to drop something if entry drop parts + for (const auto & part_to_drop : entry_representation.dropped_parts) { - /// We don't need to mutate part if it's covered by DROP_RANGE - if (!drop_ranges.hasDropRange(part_info)) - result.push_back(covered_part_name); + auto part_to_drop_info = MergeTreePartInfo::fromPartName(part_to_drop, format_version); + if (part_to_drop_info.partition_id == partition_id) + result.removePartAndCoveredParts(part_to_drop); + } + + /// After we have to add parts if entry adds them + for (const auto & part_to_add : entry_representation.produced_parts) + { + auto part_to_add_info = MergeTreePartInfo::fromPartName(part_to_add, format_version); + if (part_to_add_info.partition_id == partition_id && part_to_add_info.getDataVersion() < block_num) + result.add(part_to_add); } } } @@ -858,20 +957,13 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C LOG_TRACE(log, "Adding mutation {} for partition {} for all block numbers less than {}", entry->znode_name, partition_id, block_num); } - /// Initialize `mutation.parts_to_do`. - /// We need to mutate all parts in `current_parts` and all parts that will appear after queue entries execution. - /// So, we need to mutate all parts in virtual_parts (with the corresponding block numbers). - Strings virtual_parts_to_mutate = getPartNamesToMutate(*entry, virtual_parts, drop_ranges); - for (const String & current_part_to_mutate : virtual_parts_to_mutate) - { - assert(MergeTreePartInfo::fromPartName(current_part_to_mutate, format_version).level < MergeTreePartInfo::MAX_LEVEL); - mutation.parts_to_do.add(current_part_to_mutate); - } + /// Initialize `mutation.parts_to_do`. We cannot use only current_parts + virtual_parts here so we + /// traverse all the queue and build correct state of parts_to_do. + auto queue_representation = getQueueRepresentation(queue, format_version); + mutation.parts_to_do = getPartNamesToMutate(*entry, virtual_parts, queue_representation, format_version); if (mutation.parts_to_do.size() == 0) - { some_mutations_are_probably_done = true; - } /// otherwise it's already done if (entry->isAlterMutation() && entry->znode_name > mutation_pointer) @@ -1680,9 +1772,9 @@ size_t ReplicatedMergeTreeQueue::countFinishedMutations() const } -ReplicatedMergeTreeMergePredicate ReplicatedMergeTreeQueue::getMergePredicate(zkutil::ZooKeeperPtr & zookeeper) +ReplicatedMergeTreeMergePredicate ReplicatedMergeTreeQueue::getMergePredicate(zkutil::ZooKeeperPtr & zookeeper, PartitionIdsHint && partition_ids_hint) { - return ReplicatedMergeTreeMergePredicate(*this, zookeeper); + return ReplicatedMergeTreeMergePredicate(*this, zookeeper, std::move(partition_ids_hint)); } @@ -1774,8 +1866,11 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep } else if (mutation.parts_to_do.size() == 0) { + /// Why it doesn't mean that mutation 100% finished? Because when we were creating part_to_do set + /// some INSERT queries could be in progress. So we have to double-check that no affected committing block + /// numbers exist and no new parts were surprisingly committed. LOG_TRACE(log, "Will check if mutation {} is done", mutation.entry->znode_name); - candidates.push_back(mutation.entry); + candidates.emplace_back(mutation.entry); } } } @@ -1785,12 +1880,20 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep else LOG_DEBUG(log, "Trying to finalize {} mutations", candidates.size()); - auto merge_pred = getMergePredicate(zookeeper); + /// We need to check committing block numbers and new parts which could be committed. + /// Actually we don't need most of predicate logic here but it all the code related to committing blocks + /// and updatating queue state is implemented there. + PartitionIdsHint partition_ids_hint; + for (const auto & candidate : candidates) + for (const auto & partitions : candidate->block_numbers) + partition_ids_hint.insert(partitions.first); + + auto merge_pred = getMergePredicate(zookeeper, std::move(partition_ids_hint)); std::vector finished; - for (const ReplicatedMergeTreeMutationEntryPtr & candidate : candidates) + for (const auto & candidate : candidates) { - if (merge_pred.isMutationFinished(*candidate)) + if (merge_pred.isMutationFinished(candidate->znode_name, candidate->block_numbers)) finished.push_back(candidate.get()); } @@ -1983,8 +2086,9 @@ ReplicatedMergeTreeQueue::QueueLocks ReplicatedMergeTreeQueue::lockQueue() } ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( - ReplicatedMergeTreeQueue & queue_, zkutil::ZooKeeperPtr & zookeeper) + ReplicatedMergeTreeQueue & queue_, zkutil::ZooKeeperPtr & zookeeper, PartitionIdsHint && partition_ids_hint_) : queue(queue_) + , partition_ids_hint(std::move(partition_ids_hint_)) , prev_virtual_parts(queue.format_version) { { @@ -1996,7 +2100,15 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( auto quorum_status_future = zookeeper->asyncTryGet(fs::path(queue.zookeeper_path) / "quorum" / "status"); /// Load current inserts - Strings partitions = zookeeper->getChildren(fs::path(queue.zookeeper_path) / "block_numbers"); + /// Hint avoids listing partitions that we don't really need. + /// Dropped (or cleaned up by TTL) partitions are never removed from ZK, + /// so without hint it can do a few thousands requests (if not using MultiRead). + Strings partitions; + if (partition_ids_hint.empty()) + partitions = zookeeper->getChildren(fs::path(queue.zookeeper_path) / "block_numbers"); + else + std::copy(partition_ids_hint.begin(), partition_ids_hint.end(), std::back_inserter(partitions)); + std::vector paths; paths.reserve(partitions.size()); for (const String & partition : partitions) @@ -2128,6 +2240,13 @@ bool ReplicatedMergeTreeMergePredicate::canMergeTwoParts( if (left_max_block + 1 < right_min_block) { + if (!partition_ids_hint.empty() && !partition_ids_hint.contains(left->info.partition_id)) + { + if (out_reason) + *out_reason = fmt::format("Uncommitted block were not loaded for unexpected partition {}", left->info.partition_id); + return false; + } + auto committing_blocks_in_partition = committing_blocks.find(left->info.partition_id); if (committing_blocks_in_partition != committing_blocks.end()) { @@ -2312,13 +2431,18 @@ std::optional> ReplicatedMergeTreeMergePredicate::getDesir } -bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMergeTreeMutationEntry & mutation) const +bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const std::string & znode_name, const std::map & block_numbers) const { - for (const auto & kv : mutation.block_numbers) + /// Check committing block numbers, maybe some affected inserts + /// still not written to disk and committed to ZK. + for (const auto & kv : block_numbers) { const String & partition_id = kv.first; Int64 block_num = kv.second; + if (!partition_ids_hint.empty() && !partition_ids_hint.contains(partition_id)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Partition id {} was not provided as hint, it's a bug", partition_id); + auto partition_it = committing_blocks.find(partition_id); if (partition_it != committing_blocks.end()) { @@ -2326,24 +2450,28 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge partition_it->second.begin(), partition_it->second.lower_bound(block_num)); if (blocks_count) { - LOG_TRACE(queue.log, "Mutation {} is not done yet because in partition ID {} there are still {} uncommitted blocks.", mutation.znode_name, partition_id, blocks_count); + LOG_TRACE(queue.log, "Mutation {} is not done yet because in partition ID {} there are still {} uncommitted blocks.", znode_name, partition_id, blocks_count); return false; } } } + std::lock_guard lock(queue.state_mutex); + /// When we creating predicate we have updated the queue. Some committing inserts can now be committed so + /// we check parts_to_do one more time. Also this code is async so mutation actually could be deleted from memory. + if (auto it = queue.mutations_by_znode.find(znode_name); it != queue.mutations_by_znode.end()) { - std::lock_guard lock(queue.state_mutex); + if (it->second.parts_to_do.size() == 0) + return true; - size_t suddenly_appeared_parts = getPartNamesToMutate(mutation, queue.virtual_parts, queue.drop_ranges).size(); - if (suddenly_appeared_parts) - { - LOG_TRACE(queue.log, "Mutation {} is not done yet because {} parts to mutate suddenly appeared.", mutation.znode_name, suddenly_appeared_parts); - return false; - } + LOG_TRACE(queue.log, "Mutation {} is not done because some parts [{}] were just committed", znode_name, fmt::join(it->second.parts_to_do.getParts(), ", ")); + return false; + } + else + { + LOG_TRACE(queue.log, "Mutation {} is done because it doesn't exist anymore", znode_name); + return true; } - - return true; } bool ReplicatedMergeTreeMergePredicate::hasDropRange(const MergeTreePartInfo & new_drop_range_info) const diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 32421f91b04..36f1ee07ad4 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -25,6 +25,7 @@ class MergeTreeDataMergerMutator; class ReplicatedMergeTreeMergePredicate; class ReplicatedMergeTreeMergeStrategyPicker; +using PartitionIdsHint = std::unordered_set; class ReplicatedMergeTreeQueue { @@ -382,7 +383,7 @@ public: size_t countFinishedMutations() const; /// Returns functor which used by MergeTreeMergerMutator to select parts for merge - ReplicatedMergeTreeMergePredicate getMergePredicate(zkutil::ZooKeeperPtr & zookeeper); + ReplicatedMergeTreeMergePredicate getMergePredicate(zkutil::ZooKeeperPtr & zookeeper, PartitionIdsHint && partition_ids_hint); /// Return the version (block number) of the last mutation that we don't need to apply to the part /// with getDataVersion() == data_version. (Either this mutation was already applied or the part @@ -486,7 +487,7 @@ public: class ReplicatedMergeTreeMergePredicate { public: - ReplicatedMergeTreeMergePredicate(ReplicatedMergeTreeQueue & queue_, zkutil::ZooKeeperPtr & zookeeper); + ReplicatedMergeTreeMergePredicate(ReplicatedMergeTreeQueue & queue_, zkutil::ZooKeeperPtr & zookeeper, PartitionIdsHint && partition_ids_hint_); /// Depending on the existence of left part checks a merge predicate for two parts or for single part. bool operator()(const MergeTreeData::DataPartPtr & left, @@ -517,7 +518,7 @@ public: /// don't glue them together. Alter is rare operation, so it shouldn't affect performance. std::optional> getDesiredMutationVersion(const MergeTreeData::DataPartPtr & part) const; - bool isMutationFinished(const ReplicatedMergeTreeMutationEntry & mutation) const; + bool isMutationFinished(const std::string & znode_name, const std::map & block_numbers) const; /// The version of "log" node that is used to check that no new merges have appeared. int32_t getVersion() const { return merges_version; } @@ -531,6 +532,8 @@ public: private: const ReplicatedMergeTreeQueue & queue; + PartitionIdsHint partition_ids_hint; + /// A snapshot of active parts that would appear if the replica executes all log entries in its queue. ActiveDataPartSet prev_virtual_parts; /// partition ID -> block numbers of the inserts and mutations that are about to commit diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp index dea2553700b..5a8e250a972 100644 --- a/src/Storages/NATS/StorageNATS.cpp +++ b/src/Storages/NATS/StorageNATS.cpp @@ -535,24 +535,24 @@ bool StorageNATS::isSubjectInSubscriptions(const std::string & subject) bool StorageNATS::checkDependencies(const StorageID & table_id) { // Check if all dependencies are attached - auto dependencies = DatabaseCatalog::instance().getDependencies(table_id); - if (dependencies.empty()) + auto view_ids = DatabaseCatalog::instance().getDependentViews(table_id); + if (view_ids.empty()) return true; // Check the dependencies are ready? - for (const auto & db_tab : dependencies) + for (const auto & view_id : view_ids) { - auto table = DatabaseCatalog::instance().tryGetTable(db_tab, getContext()); - if (!table) + auto view = DatabaseCatalog::instance().tryGetTable(view_id, getContext()); + if (!view) return false; // If it materialized view, check it's target table - auto * materialized_view = dynamic_cast(table.get()); + auto * materialized_view = dynamic_cast(view.get()); if (materialized_view && !materialized_view->tryGetTargetTable()) return false; // Check all its dependencies - if (!checkDependencies(db_tab)) + if (!checkDependencies(view_id)) return false; } @@ -568,10 +568,10 @@ void StorageNATS::streamingToViewsFunc() auto table_id = getStorageID(); // Check if at least one direct dependency is attached - size_t dependencies_count = DatabaseCatalog::instance().getDependencies(table_id).size(); + size_t num_views = DatabaseCatalog::instance().getDependentViews(table_id).size(); bool nats_connected = connection->isConnected() || connection->reconnect(); - if (dependencies_count && nats_connected) + if (num_views && nats_connected) { auto start_time = std::chrono::steady_clock::now(); @@ -583,7 +583,7 @@ void StorageNATS::streamingToViewsFunc() if (!checkDependencies(table_id)) break; - LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count); + LOG_DEBUG(log, "Started streaming to {} attached views", num_views); if (streamToViews()) { diff --git a/src/Storages/NamedCollectionConfiguration.cpp b/src/Storages/NamedCollectionConfiguration.cpp new file mode 100644 index 00000000000..b0e7bdce32a --- /dev/null +++ b/src/Storages/NamedCollectionConfiguration.cpp @@ -0,0 +1,174 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int NOT_IMPLEMENTED; +} + +namespace NamedCollectionConfiguration +{ + +template T getConfigValue( + const Poco::Util::AbstractConfiguration & config, + const std::string & path) +{ + return getConfigValueOrDefault(config, path); +} + +template T getConfigValueOrDefault( + const Poco::Util::AbstractConfiguration & config, + const std::string & path, + const T * default_value) +{ + if (!config.has(path)) + { + if (!default_value) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "No such key `{}`", path); + return *default_value; + } + + if constexpr (std::is_same_v) + return config.getString(path); + else if constexpr (std::is_same_v) + return config.getUInt64(path); + else if constexpr (std::is_same_v) + return config.getInt64(path); + else if constexpr (std::is_same_v) + return config.getDouble(path); + else + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Unsupported type in getConfigValueOrDefault(). " + "Supported types are String, UInt64, Int64, Float64"); +} + +template void setConfigValue( + Poco::Util::AbstractConfiguration & config, + const std::string & path, + const T & value, + bool update) +{ + if (!update && config.has(path)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Key `{}` already exists", path); + + if constexpr (std::is_same_v) + config.setString(path, value); + else if constexpr (std::is_same_v) + config.setUInt64(path, value); + else if constexpr (std::is_same_v) + config.setInt64(path, value); + else if constexpr (std::is_same_v) + config.setDouble(path, value); + else + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Unsupported type in setConfigValue(). " + "Supported types are String, UInt64, Int64, Float64"); +} + +template void copyConfigValue( + const Poco::Util::AbstractConfiguration & from_config, + const std::string & from_path, + Poco::Util::AbstractConfiguration & to_config, + const std::string & to_path) +{ + if (!from_config.has(from_path)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "No such key `{}`", from_path); + + if (to_config.has(to_path)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Key `{}` already exists", to_path); + + if constexpr (std::is_same_v) + to_config.setString(to_path, from_config.getString(from_path)); + else if constexpr (std::is_same_v) + to_config.setUInt64(to_path, from_config.getUInt64(from_path)); + else if constexpr (std::is_same_v) + to_config.setInt64(to_path, from_config.getInt64(from_path)); + else if constexpr (std::is_same_v) + to_config.setDouble(to_path, from_config.getDouble(from_path)); + else + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Unsupported type in copyConfigValue(). " + "Supported types are String, UInt64, Int64, Float64"); +} + +void removeConfigValue( + Poco::Util::AbstractConfiguration & config, + const std::string & path) +{ + if (!config.has(path)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "No such key `{}`", path); + config.remove(path); +} + +ConfigurationPtr createEmptyConfiguration(const std::string & root_name) +{ + using DocumentPtr = Poco::AutoPtr; + using ElementPtr = Poco::AutoPtr; + + DocumentPtr xml_document(new Poco::XML::Document()); + ElementPtr root_element(xml_document->createElement(root_name)); + xml_document->appendChild(root_element); + + ConfigurationPtr config(new Poco::Util::XMLConfiguration(xml_document)); + return config; +} + +ConfigurationPtr createConfiguration(const std::string & root_name, const SettingsChanges & settings) +{ + namespace Configuration = NamedCollectionConfiguration; + + auto config = Configuration::createEmptyConfiguration(root_name); + for (const auto & [name, value] : settings) + Configuration::setConfigValue(*config, name, convertFieldToString(value)); + + return config; +} + +template String getConfigValue(const Poco::Util::AbstractConfiguration & config, + const std::string & path); +template UInt64 getConfigValue(const Poco::Util::AbstractConfiguration & config, + const std::string & path); +template Int64 getConfigValue(const Poco::Util::AbstractConfiguration & config, + const std::string & path); +template Float64 getConfigValue(const Poco::Util::AbstractConfiguration & config, + const std::string & path); + +template String getConfigValueOrDefault(const Poco::Util::AbstractConfiguration & config, + const std::string & path, const String * default_value); +template UInt64 getConfigValueOrDefault(const Poco::Util::AbstractConfiguration & config, + const std::string & path, const UInt64 * default_value); +template Int64 getConfigValueOrDefault(const Poco::Util::AbstractConfiguration & config, + const std::string & path, const Int64 * default_value); +template Float64 getConfigValueOrDefault(const Poco::Util::AbstractConfiguration & config, + const std::string & path, const Float64 * default_value); + +template void setConfigValue(Poco::Util::AbstractConfiguration & config, + const std::string & path, const String & value, bool update); +template void setConfigValue(Poco::Util::AbstractConfiguration & config, + const std::string & path, const UInt64 & value, bool update); +template void setConfigValue(Poco::Util::AbstractConfiguration & config, + const std::string & path, const Int64 & value, bool update); +template void setConfigValue(Poco::Util::AbstractConfiguration & config, + const std::string & path, const Float64 & value, bool update); + +template void copyConfigValue(const Poco::Util::AbstractConfiguration & from_config, const std::string & from_path, + Poco::Util::AbstractConfiguration & to_config, const std::string & to_path); +template void copyConfigValue(const Poco::Util::AbstractConfiguration & from_config, const std::string & from_path, + Poco::Util::AbstractConfiguration & to_config, const std::string & to_path); +template void copyConfigValue(const Poco::Util::AbstractConfiguration & from_config, const std::string & from_path, + Poco::Util::AbstractConfiguration & to_config, const std::string & to_path); +template void copyConfigValue(const Poco::Util::AbstractConfiguration & from_config, const std::string & from_path, + Poco::Util::AbstractConfiguration & to_config, const std::string & to_path); +} + +} diff --git a/src/Storages/NamedCollectionConfiguration.h b/src/Storages/NamedCollectionConfiguration.h new file mode 100644 index 00000000000..7478dcf2d9a --- /dev/null +++ b/src/Storages/NamedCollectionConfiguration.h @@ -0,0 +1,44 @@ +#pragma once +#include + +namespace DB +{ + +using ConfigurationPtr = Poco::AutoPtr; +class SettingsChanges; + +namespace NamedCollectionConfiguration +{ + +ConfigurationPtr createEmptyConfiguration(const std::string & root_name); + +template T getConfigValue( + const Poco::Util::AbstractConfiguration & config, + const std::string & path); + +template T getConfigValueOrDefault( + const Poco::Util::AbstractConfiguration & config, + const std::string & path, + const T * default_value = nullptr); + +template void setConfigValue( + Poco::Util::AbstractConfiguration & config, + const std::string & path, + const T & value, + bool update = false); + +template void copyConfigValue( + const Poco::Util::AbstractConfiguration & from_config, + const std::string & from_path, + Poco::Util::AbstractConfiguration & to_config, + const std::string & to_path); + +void removeConfigValue( + Poco::Util::AbstractConfiguration & config, + const std::string & path); + +ConfigurationPtr createConfiguration(const std::string & root_name, const SettingsChanges & settings); + +} + +} diff --git a/src/Storages/NamedCollectionUtils.cpp b/src/Storages/NamedCollectionUtils.cpp new file mode 100644 index 00000000000..75d5aace664 --- /dev/null +++ b/src/Storages/NamedCollectionUtils.cpp @@ -0,0 +1,434 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace fs = std::filesystem; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NAMED_COLLECTION_ALREADY_EXISTS; + extern const int NAMED_COLLECTION_DOESNT_EXIST; + extern const int BAD_ARGUMENTS; +} + +namespace NamedCollectionUtils +{ + +class LoadFromConfig +{ +private: + const Poco::Util::AbstractConfiguration & config; + +public: + explicit LoadFromConfig(const Poco::Util::AbstractConfiguration & config_) + : config(config_) {} + + std::vector listCollections() const + { + Poco::Util::AbstractConfiguration::Keys collections_names; + config.keys(NAMED_COLLECTIONS_CONFIG_PREFIX, collections_names); + return collections_names; + } + + NamedCollectionsMap getAll() const + { + NamedCollectionsMap result; + for (const auto & collection_name : listCollections()) + { + if (result.contains(collection_name)) + { + throw Exception( + ErrorCodes::NAMED_COLLECTION_ALREADY_EXISTS, + "Found duplicate named collection `{}`", + collection_name); + } + result.emplace(collection_name, get(collection_name)); + } + return result; + } + + MutableNamedCollectionPtr get(const std::string & collection_name) const + { + const auto collection_prefix = getCollectionPrefix(collection_name); + std::queue enumerate_input; + std::set enumerate_result; + + enumerate_input.push(collection_prefix); + collectKeys(config, std::move(enumerate_input), enumerate_result); + + /// Collection does not have any keys. + /// (`enumerate_result` == ). + const bool collection_is_empty = enumerate_result.size() == 1 + && *enumerate_result.begin() == collection_prefix; + std::set keys; + if (!collection_is_empty) + { + /// Skip collection prefix and add +1 to avoid '.' in the beginning. + for (const auto & path : enumerate_result) + keys.emplace(path.substr(collection_prefix.size() + 1)); + } + + return NamedCollection::create( + config, collection_name, collection_prefix, keys, SourceId::CONFIG, /* is_mutable */false); + } + +private: + static constexpr auto NAMED_COLLECTIONS_CONFIG_PREFIX = "named_collections"; + + static std::string getCollectionPrefix(const std::string & collection_name) + { + return fmt::format("{}.{}", NAMED_COLLECTIONS_CONFIG_PREFIX, collection_name); + } + + /// Enumerate keys paths of the config recursively. + /// E.g. if `enumerate_paths` = {"root.key1"} and config like + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// the `result` will contain two strings: "root.key1.key2" and "root.key1.key3.key4" + static void collectKeys( + const Poco::Util::AbstractConfiguration & config, + std::queue enumerate_paths, + std::set & result) + { + if (enumerate_paths.empty()) + return; + + auto initial_paths = std::move(enumerate_paths); + enumerate_paths = {}; + while (!initial_paths.empty()) + { + auto path = initial_paths.front(); + initial_paths.pop(); + + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(path, keys); + + if (keys.empty()) + { + result.insert(path); + } + else + { + for (const auto & key : keys) + enumerate_paths.emplace(path + '.' + key); + } + } + + collectKeys(config, enumerate_paths, result); + } +}; + + +class LoadFromSQL : private WithContext +{ +private: + const std::string metadata_path; + +public: + explicit LoadFromSQL(ContextPtr context_) + : WithContext(context_) + , metadata_path( + fs::canonical(context_->getPath()) / NAMED_COLLECTIONS_METADATA_DIRECTORY) + { + if (fs::exists(metadata_path)) + cleanUp(); + else + fs::create_directories(metadata_path); + } + + std::vector listCollections() const + { + std::vector collection_names; + fs::directory_iterator it{metadata_path}; + for (; it != fs::directory_iterator{}; ++it) + { + const auto & current_path = it->path(); + if (current_path.extension() == ".sql") + { + collection_names.push_back(it->path().stem()); + } + else + { + LOG_WARNING( + &Poco::Logger::get("NamedCollectionsLoadFromSQL"), + "Unexpected file {} in named collections directory", + current_path.filename().string()); + } + } + return collection_names; + } + + NamedCollectionsMap getAll() const + { + NamedCollectionsMap result; + for (const auto & collection_name : listCollections()) + { + if (result.contains(collection_name)) + { + throw Exception( + ErrorCodes::NAMED_COLLECTION_ALREADY_EXISTS, + "Found duplicate named collection `{}`", + collection_name); + } + result.emplace(collection_name, get(collection_name)); + } + return result; + } + + MutableNamedCollectionPtr get(const std::string & collection_name) const + { + const auto query = readCreateQueryFromMetadata( + getMetadataPath(collection_name), + getContext()->getSettingsRef()); + return createNamedCollectionFromAST(query); + } + + MutableNamedCollectionPtr create(const ASTCreateNamedCollectionQuery & query) + { + writeCreateQueryToMetadata( + query, + getMetadataPath(query.collection_name), + getContext()->getSettingsRef()); + + return createNamedCollectionFromAST(query); + } + + void update(const ASTAlterNamedCollectionQuery & query) + { + const auto path = getMetadataPath(query.collection_name); + auto create_query = readCreateQueryFromMetadata(path, getContext()->getSettings()); + + std::unordered_map result_changes_map; + for (const auto & [name, value] : query.changes) + { + auto [it, inserted] = result_changes_map.emplace(name, value); + if (!inserted) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Value with key `{}` is used twice in the SET query", + name, query.collection_name); + } + } + + for (const auto & [name, value] : create_query.changes) + result_changes_map.emplace(name, value); + + for (const auto & delete_key : query.delete_keys) + { + auto it = result_changes_map.find(delete_key); + if (it == result_changes_map.end()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot delete key `{}` because it does not exist in collection", + delete_key); + } + else + result_changes_map.erase(it); + } + + create_query.changes.clear(); + for (const auto & [name, value] : result_changes_map) + create_query.changes.emplace_back(name, value); + + writeCreateQueryToMetadata( + create_query, + getMetadataPath(query.collection_name), + getContext()->getSettingsRef(), + true); + } + + void remove(const std::string & collection_name) + { + if (!removeIfExists(collection_name)) + { + throw Exception( + ErrorCodes::NAMED_COLLECTION_DOESNT_EXIST, + "Cannot remove collection `{}`, because it doesn't exist", + collection_name); + } + } + + bool removeIfExists(const std::string & collection_name) + { + auto collection_path = getMetadataPath(collection_name); + if (fs::exists(collection_path)) + { + fs::remove(collection_path); + return true; + } + return false; + } + +private: + static constexpr auto NAMED_COLLECTIONS_METADATA_DIRECTORY = "named_collections"; + + static MutableNamedCollectionPtr createNamedCollectionFromAST( + const ASTCreateNamedCollectionQuery & query) + { + const auto & collection_name = query.collection_name; + const auto config = NamedCollectionConfiguration::createConfiguration( + collection_name, query.changes); + + std::set keys; + for (const auto & [name, _] : query.changes) + keys.insert(name); + + return NamedCollection::create( + *config, collection_name, "", keys, SourceId::SQL, /* is_mutable */true); + } + + std::string getMetadataPath(const std::string & collection_name) const + { + return fs::path(metadata_path) / (escapeForFileName(collection_name) + ".sql"); + } + + /// Delete .tmp files. They could be left undeleted in case of + /// some exception or abrupt server restart. + void cleanUp() + { + fs::directory_iterator it{metadata_path}; + std::vector files_to_remove; + for (; it != fs::directory_iterator{}; ++it) + { + const auto & current_path = it->path(); + if (current_path.extension() == ".tmp") + files_to_remove.push_back(current_path); + } + for (const auto & file : files_to_remove) + fs::remove(file); + } + + static ASTCreateNamedCollectionQuery readCreateQueryFromMetadata( + const std::string & path, + const Settings & settings) + { + ReadBufferFromFile in(path); + std::string query; + readStringUntilEOF(query, in); + + ParserCreateNamedCollectionQuery parser; + auto ast = parseQuery(parser, query, "in file " + path, 0, settings.max_parser_depth); + const auto & create_query = ast->as(); + return create_query; + } + + static void writeCreateQueryToMetadata( + const ASTCreateNamedCollectionQuery & query, + const std::string & path, + const Settings & settings, + bool replace = false) + { + if (!replace && fs::exists(path)) + { + throw Exception( + ErrorCodes::NAMED_COLLECTION_ALREADY_EXISTS, + "Metadata file {} for named collection already exists", + path); + } + + auto tmp_path = path + ".tmp"; + String formatted_query = serializeAST(query); + WriteBufferFromFile out(tmp_path, formatted_query.size(), O_WRONLY | O_CREAT | O_EXCL); + writeString(formatted_query, out); + + out.next(); + if (settings.fsync_metadata) + out.sync(); + out.close(); + + fs::rename(tmp_path, path); + } +}; + +std::unique_lock lockNamedCollectionsTransaction() +{ + static std::mutex transaction_lock; + return std::unique_lock(transaction_lock); +} + +void loadFromConfig(const Poco::Util::AbstractConfiguration & config) +{ + auto lock = lockNamedCollectionsTransaction(); + NamedCollectionFactory::instance().add(LoadFromConfig(config).getAll()); +} + +void reloadFromConfig(const Poco::Util::AbstractConfiguration & config) +{ + auto lock = lockNamedCollectionsTransaction(); + auto collections = LoadFromConfig(config).getAll(); + auto & instance = NamedCollectionFactory::instance(); + instance.removeById(SourceId::CONFIG); + instance.add(collections); +} + +void loadFromSQL(ContextPtr context) +{ + auto lock = lockNamedCollectionsTransaction(); + NamedCollectionFactory::instance().add(LoadFromSQL(context).getAll()); +} + +void removeFromSQL(const std::string & collection_name, ContextPtr context) +{ + auto lock = lockNamedCollectionsTransaction(); + LoadFromSQL(context).remove(collection_name); + NamedCollectionFactory::instance().remove(collection_name); +} + +void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr context) +{ + auto lock = lockNamedCollectionsTransaction(); + LoadFromSQL(context).removeIfExists(collection_name); + NamedCollectionFactory::instance().removeIfExists(collection_name); +} + +void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr context) +{ + auto lock = lockNamedCollectionsTransaction(); + NamedCollectionFactory::instance().add(query.collection_name, LoadFromSQL(context).create(query)); +} + +void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context) +{ + auto lock = lockNamedCollectionsTransaction(); + LoadFromSQL(context).update(query); + + auto collection = NamedCollectionFactory::instance().getMutable(query.collection_name); + auto collection_lock = collection->lock(); + + for (const auto & [name, value] : query.changes) + collection->setOrUpdate(name, convertFieldToString(value)); + + for (const auto & key : query.delete_keys) + collection->remove(key); +} + +} + +} diff --git a/src/Storages/NamedCollectionUtils.h b/src/Storages/NamedCollectionUtils.h new file mode 100644 index 00000000000..8befc9cac3c --- /dev/null +++ b/src/Storages/NamedCollectionUtils.h @@ -0,0 +1,40 @@ +#pragma once +#include + +namespace Poco { namespace Util { class AbstractConfiguration; } } + +namespace DB +{ + +class ASTCreateNamedCollectionQuery; +class ASTAlterNamedCollectionQuery; + +namespace NamedCollectionUtils +{ + +enum class SourceId +{ + NONE = 0, + CONFIG = 1, + SQL = 2, +}; + +void loadFromConfig(const Poco::Util::AbstractConfiguration & config); +void reloadFromConfig(const Poco::Util::AbstractConfiguration & config); + +/// Load named collections from `context->getPath() / named_collections /`. +void loadFromSQL(ContextPtr context); + +/// Remove collection as well as its metadata from `context->getPath() / named_collections /`. +void removeFromSQL(const std::string & collection_name, ContextPtr context); +void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr context); + +/// Create a new collection from AST and put it to `context->getPath() / named_collections /`. +void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr context); + +/// Update definition of already existing collection from AST and update result in `context->getPath() / named_collections /`. +void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context); + +} + +} diff --git a/src/Storages/NamedCollections.cpp b/src/Storages/NamedCollections.cpp index 67847635f3f..d90225547ac 100644 --- a/src/Storages/NamedCollections.cpp +++ b/src/Storages/NamedCollections.cpp @@ -1,17 +1,11 @@ #include "NamedCollections.h" -#include -#include -#include #include -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include #include @@ -20,66 +14,13 @@ namespace DB namespace ErrorCodes { - extern const int UNKNOWN_NAMED_COLLECTION; + extern const int NAMED_COLLECTION_DOESNT_EXIST; extern const int NAMED_COLLECTION_ALREADY_EXISTS; - extern const int BAD_ARGUMENTS; - extern const int NOT_IMPLEMENTED; - extern const int LOGICAL_ERROR; + extern const int NAMED_COLLECTION_IS_IMMUTABLE; } -namespace -{ - constexpr auto NAMED_COLLECTIONS_CONFIG_PREFIX = "named_collections"; +namespace Configuration = NamedCollectionConfiguration; - std::string getCollectionPrefix(const std::string & collection_name) - { - return fmt::format("{}.{}", NAMED_COLLECTIONS_CONFIG_PREFIX, collection_name); - } - - /// Enumerate keys paths of the config recursively. - /// E.g. if `enumerate_paths` = {"root.key1"} and config like - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// the `result` will contain two strings: "root.key1.key2" and "root.key1.key3.key4" - void collectKeys( - const Poco::Util::AbstractConfiguration & config, - std::queue enumerate_paths, - std::set & result) - { - if (enumerate_paths.empty()) - return; - - auto initial_paths = std::move(enumerate_paths); - enumerate_paths = {}; - while (!initial_paths.empty()) - { - auto path = initial_paths.front(); - initial_paths.pop(); - - Poco::Util::AbstractConfiguration::Keys keys; - config.keys(path, keys); - - if (keys.empty()) - { - result.insert(path); - } - else - { - for (const auto & key : keys) - enumerate_paths.emplace(path + '.' + key); - } - } - - collectKeys(config, enumerate_paths, result); - } -} NamedCollectionFactory & NamedCollectionFactory::instance() { @@ -87,38 +28,6 @@ NamedCollectionFactory & NamedCollectionFactory::instance() return instance; } -void NamedCollectionFactory::initialize(const Poco::Util::AbstractConfiguration & config_) -{ - std::lock_guard lock(mutex); - if (is_initialized) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Named collection factory already initialized"); - } - - config = &config_; - is_initialized = true; -} - -void NamedCollectionFactory::reload(const Poco::Util::AbstractConfiguration & config_) -{ - std::lock_guard lock(mutex); - config = &config_; - loaded_named_collections.clear(); -} - -void NamedCollectionFactory::assertInitialized( - std::lock_guard & /* lock */) const -{ - if (!is_initialized) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Named collection factory must be initialized before being used"); - } -} - bool NamedCollectionFactory::exists(const std::string & collection_name) const { std::lock_guard lock(mutex); @@ -127,62 +36,84 @@ bool NamedCollectionFactory::exists(const std::string & collection_name) const bool NamedCollectionFactory::existsUnlocked( const std::string & collection_name, - std::lock_guard & lock) const + std::lock_guard & /* lock */) const { - assertInitialized(lock); - /// Named collections can be added via SQL command or via config. - /// Named collections from config are loaded on first access, - /// therefore it might not be in `named_collections` map yet. - return loaded_named_collections.contains(collection_name) - || config->has(getCollectionPrefix(collection_name)); + return loaded_named_collections.contains(collection_name); } NamedCollectionPtr NamedCollectionFactory::get(const std::string & collection_name) const { std::lock_guard lock(mutex); - assertInitialized(lock); - - if (!existsUnlocked(collection_name, lock)) + auto collection = tryGetUnlocked(collection_name, lock); + if (!collection) { throw Exception( - ErrorCodes::UNKNOWN_NAMED_COLLECTION, + ErrorCodes::NAMED_COLLECTION_DOESNT_EXIST, "There is no named collection `{}`", collection_name); } - - return getImpl(collection_name, lock); + return collection; } NamedCollectionPtr NamedCollectionFactory::tryGet(const std::string & collection_name) const { std::lock_guard lock(mutex); - assertInitialized(lock); - - if (!existsUnlocked(collection_name, lock)) - return nullptr; - - return getImpl(collection_name, lock); + return tryGetUnlocked(collection_name, lock); } -NamedCollectionPtr NamedCollectionFactory::getImpl( +MutableNamedCollectionPtr NamedCollectionFactory::getMutable( + const std::string & collection_name) const +{ + std::lock_guard lock(mutex); + auto collection = tryGetUnlocked(collection_name, lock); + if (!collection) + { + throw Exception( + ErrorCodes::NAMED_COLLECTION_DOESNT_EXIST, + "There is no named collection `{}`", + collection_name); + } + else if (!collection->isMutable()) + { + throw Exception( + ErrorCodes::NAMED_COLLECTION_IS_IMMUTABLE, + "Cannot get collection `{}` for modification, " + "because collection was defined as immutable", + collection_name); + } + return collection; +} + +MutableNamedCollectionPtr NamedCollectionFactory::tryGetUnlocked( const std::string & collection_name, std::lock_guard & /* lock */) const { auto it = loaded_named_collections.find(collection_name); if (it == loaded_named_collections.end()) - { - it = loaded_named_collections.emplace( - collection_name, - NamedCollection::create(*config, collection_name)).first; - } + return nullptr; return it->second; } void NamedCollectionFactory::add( const std::string & collection_name, - NamedCollectionPtr collection) + MutableNamedCollectionPtr collection) { std::lock_guard lock(mutex); + return addUnlocked(collection_name, collection, lock); +} + +void NamedCollectionFactory::add(NamedCollectionsMap collections) +{ + std::lock_guard lock(mutex); + for (const auto & [collection_name, collection] : collections) + addUnlocked(collection_name, collection, lock); +} + +void NamedCollectionFactory::addUnlocked( + const std::string & collection_name, + MutableNamedCollectionPtr collection, + std::lock_guard & /* lock */) +{ auto [it, inserted] = loaded_named_collections.emplace(collection_name, collection); if (!inserted) { @@ -196,93 +127,104 @@ void NamedCollectionFactory::add( void NamedCollectionFactory::remove(const std::string & collection_name) { std::lock_guard lock(mutex); - assertInitialized(lock); - - if (!existsUnlocked(collection_name, lock)) + bool removed = removeIfExistsUnlocked(collection_name, lock); + if (!removed) { throw Exception( - ErrorCodes::UNKNOWN_NAMED_COLLECTION, + ErrorCodes::NAMED_COLLECTION_DOESNT_EXIST, "There is no named collection `{}`", collection_name); } - - if (config->has(collection_name)) - { - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - "Collection {} is defined in config and cannot be removed", - collection_name); - } - - [[maybe_unused]] auto removed = loaded_named_collections.erase(collection_name); - assert(removed); } -NamedCollectionFactory::NamedCollections NamedCollectionFactory::getAll() const +void NamedCollectionFactory::removeIfExists(const std::string & collection_name) { std::lock_guard lock(mutex); - assertInitialized(lock); + removeIfExistsUnlocked(collection_name, lock); +} - NamedCollections result(loaded_named_collections); +bool NamedCollectionFactory::removeIfExistsUnlocked( + const std::string & collection_name, + std::lock_guard & lock) +{ + auto collection = tryGetUnlocked(collection_name, lock); + if (!collection) + return false; - Poco::Util::AbstractConfiguration::Keys config_collections_names; - config->keys(NAMED_COLLECTIONS_CONFIG_PREFIX, config_collections_names); - - for (const auto & collection_name : config_collections_names) + if (!collection->isMutable()) { - if (result.contains(collection_name)) - continue; - - result.emplace(collection_name, NamedCollection::create(*config, collection_name)); + throw Exception( + ErrorCodes::NAMED_COLLECTION_IS_IMMUTABLE, + "Cannot get collection `{}` for modification, " + "because collection was defined as immutable", + collection_name); } + loaded_named_collections.erase(collection_name); + return true; +} - return result; +void NamedCollectionFactory::removeById(NamedCollectionUtils::SourceId id) +{ + std::lock_guard lock(mutex); + std::erase_if( + loaded_named_collections, + [&](const auto & value) { return value.second->getSourceId() == id; }); +} + +NamedCollectionsMap NamedCollectionFactory::getAll() const +{ + std::lock_guard lock(mutex); + return loaded_named_collections; } class NamedCollection::Impl { private: - using ConfigurationPtr = Poco::AutoPtr; - - /// Named collection configuration - /// - /// ... - /// ConfigurationPtr config; Keys keys; + Impl(ConfigurationPtr config_, const Keys & keys_) : config(config_) , keys(keys_) {} + public: - Impl(const Poco::Util::AbstractConfiguration & config_, - const std::string & collection_name_, - const Keys & keys_) - : config(createEmptyConfiguration(collection_name_)) - , keys(keys_) + static ImplPtr create( + const Poco::Util::AbstractConfiguration & config, + const std::string & collection_name, + const std::string & collection_path, + const Keys & keys) { - auto collection_path = getCollectionPrefix(collection_name_); + auto collection_config = NamedCollectionConfiguration::createEmptyConfiguration(collection_name); for (const auto & key : keys) - copyConfigValue(config_, collection_path + '.' + key, *config, key); + Configuration::copyConfigValue( + config, collection_path + '.' + key, *collection_config, key); + + return std::unique_ptr(new Impl(collection_config, keys)); } template T get(const Key & key) const { - return getConfigValue(*config, key); + return Configuration::getConfigValue(*config, key); } template T getOrDefault(const Key & key, const T & default_value) const { - return getConfigValueOrDefault(*config, key, &default_value); + return Configuration::getConfigValueOrDefault(*config, key, &default_value); } template void set(const Key & key, const T & value, bool update_if_exists) { - setConfigValue(*config, key, value, update_if_exists); + Configuration::setConfigValue(*config, key, value, update_if_exists); if (!keys.contains(key)) keys.insert(key); } + ImplPtr createCopy(const std::string & collection_name_) const + { + return create(*config, collection_name_, "", keys); + } + void remove(const Key & key) { - removeConfigValue(*config, key); + Configuration::removeConfigValue(*config, key); [[maybe_unused]] auto removed = keys.erase(key); assert(removed); } @@ -292,11 +234,6 @@ public: return keys; } - ImplPtr copy() const - { - return std::make_unique(*this); - } - std::string dumpStructure() const { /// Convert a collection config like @@ -347,186 +284,108 @@ public: } return wb.str(); } - -private: - template static T getConfigValue( - const Poco::Util::AbstractConfiguration & config, - const std::string & path) - { - return getConfigValueOrDefault(config, path); - } - - template static T getConfigValueOrDefault( - const Poco::Util::AbstractConfiguration & config, - const std::string & path, - const T * default_value = nullptr) - { - if (!config.has(path)) - { - if (!default_value) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "No such key `{}`", path); - return *default_value; - } - - if constexpr (std::is_same_v) - return config.getString(path); - else if constexpr (std::is_same_v) - return config.getUInt64(path); - else if constexpr (std::is_same_v) - return config.getInt64(path); - else if constexpr (std::is_same_v) - return config.getDouble(path); - else - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - "Unsupported type in getConfigValueOrDefault(). " - "Supported types are String, UInt64, Int64, Float64"); - } - - template static void setConfigValue( - Poco::Util::AbstractConfiguration & config, - const std::string & path, - const T & value, - bool update = false) - { - if (!update && config.has(path)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Key `{}` already exists", path); - - if constexpr (std::is_same_v) - config.setString(path, value); - else if constexpr (std::is_same_v) - config.setUInt64(path, value); - else if constexpr (std::is_same_v) - config.setInt64(path, value); - else if constexpr (std::is_same_v) - config.setDouble(path, value); - else - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - "Unsupported type in setConfigValue(). " - "Supported types are String, UInt64, Int64, Float64"); - } - - template static void copyConfigValue( - const Poco::Util::AbstractConfiguration & from_config, - const std::string & from_path, - Poco::Util::AbstractConfiguration & to_config, - const std::string & to_path) - { - if (!from_config.has(from_path)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "No such key `{}`", from_path); - - if (to_config.has(to_path)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Key `{}` already exists", to_path); - - if constexpr (std::is_same_v) - to_config.setString(to_path, from_config.getString(from_path)); - else if constexpr (std::is_same_v) - to_config.setString(to_path, from_config.getString(from_path)); - else if constexpr (std::is_same_v) - to_config.setUInt64(to_path, from_config.getUInt64(from_path)); - else if constexpr (std::is_same_v) - to_config.setInt64(to_path, from_config.getInt64(from_path)); - else if constexpr (std::is_same_v) - to_config.setDouble(to_path, from_config.getDouble(from_path)); - else - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - "Unsupported type in copyConfigValue(). " - "Supported types are String, UInt64, Int64, Float64"); - } - - static void removeConfigValue( - Poco::Util::AbstractConfiguration & config, - const std::string & path) - { - if (!config.has(path)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "No such key `{}`", path); - config.remove(path); - } - - static ConfigurationPtr createEmptyConfiguration(const std::string & root_name) - { - using DocumentPtr = Poco::AutoPtr; - DocumentPtr xml_document(new Poco::XML::Document()); - xml_document->appendChild(xml_document->createElement(root_name)); - ConfigurationPtr config(new Poco::Util::XMLConfiguration(xml_document)); - return config; - } }; NamedCollection::NamedCollection( - const Poco::Util::AbstractConfiguration & config, - const std::string & collection_path, - const Keys & keys) - : NamedCollection(std::make_unique(config, collection_path, keys)) -{ -} - -NamedCollection::NamedCollection(ImplPtr pimpl_) + ImplPtr pimpl_, + const std::string & collection_name_, + SourceId source_id_, + bool is_mutable_) : pimpl(std::move(pimpl_)) + , collection_name(collection_name_) + , source_id(source_id_) + , is_mutable(is_mutable_) { } -NamedCollectionPtr NamedCollection::create( +MutableNamedCollectionPtr NamedCollection::create( const Poco::Util::AbstractConfiguration & config, - const std::string & collection_name) + const std::string & collection_name, + const std::string & collection_path, + const Keys & keys, + SourceId source_id, + bool is_mutable) { - const auto collection_prefix = getCollectionPrefix(collection_name); - std::queue enumerate_input; - std::set enumerate_result; - - enumerate_input.push(collection_prefix); - collectKeys(config, std::move(enumerate_input), enumerate_result); - - /// Collection does not have any keys. - /// (`enumerate_result` == ). - const bool collection_is_empty = enumerate_result.size() == 1; - std::set keys; - if (!collection_is_empty) - { - /// Skip collection prefix and add +1 to avoid '.' in the beginning. - for (const auto & path : enumerate_result) - keys.emplace(path.substr(collection_prefix.size() + 1)); - } - return std::make_unique(config, collection_name, keys); + auto impl = Impl::create(config, collection_name, collection_path, keys); + return std::unique_ptr( + new NamedCollection(std::move(impl), collection_name, source_id, is_mutable)); } template T NamedCollection::get(const Key & key) const { + std::lock_guard lock(mutex); return pimpl->get(key); } template T NamedCollection::getOrDefault(const Key & key, const T & default_value) const { + std::lock_guard lock(mutex); return pimpl->getOrDefault(key, default_value); } -template void NamedCollection::set(const Key & key, const T & value, bool update_if_exists) +template void NamedCollection::set(const Key & key, const T & value) { - pimpl->set(key, value, update_if_exists); + assertMutable(); + std::unique_lock lock(mutex, std::defer_lock); + if constexpr (!Locked) + lock.lock(); + pimpl->set(key, value, false); } -void NamedCollection::remove(const Key & key) +template void NamedCollection::setOrUpdate(const Key & key, const T & value) { + assertMutable(); + std::unique_lock lock(mutex, std::defer_lock); + if constexpr (!Locked) + lock.lock(); + pimpl->set(key, value, true); +} + +template void NamedCollection::remove(const Key & key) +{ + assertMutable(); + std::unique_lock lock(mutex, std::defer_lock); + if constexpr (!Locked) + lock.lock(); pimpl->remove(key); } -std::shared_ptr NamedCollection::duplicate() const +void NamedCollection::assertMutable() const { - return std::make_shared(pimpl->copy()); + if (!is_mutable) + { + throw Exception( + ErrorCodes::NAMED_COLLECTION_IS_IMMUTABLE, + "Cannot change named collection because it is immutable"); + } +} + +MutableNamedCollectionPtr NamedCollection::duplicate() const +{ + std::lock_guard lock(mutex); + auto impl = pimpl->createCopy(collection_name); + return std::unique_ptr( + new NamedCollection( + std::move(impl), collection_name, NamedCollectionUtils::SourceId::NONE, true)); } NamedCollection::Keys NamedCollection::getKeys() const { + std::lock_guard lock(mutex); return pimpl->getKeys(); } std::string NamedCollection::dumpStructure() const { + std::lock_guard lock(mutex); return pimpl->dumpStructure(); } +std::unique_lock NamedCollection::lock() +{ + return std::unique_lock(mutex); +} + template String NamedCollection::get(const NamedCollection::Key & key) const; template UInt64 NamedCollection::get(const NamedCollection::Key & key) const; template Int64 NamedCollection::get(const NamedCollection::Key & key) const; @@ -537,9 +396,25 @@ template UInt64 NamedCollection::getOrDefault(const NamedCollection::Key template Int64 NamedCollection::getOrDefault(const NamedCollection::Key & key, const Int64 & default_value) const; template Float64 NamedCollection::getOrDefault(const NamedCollection::Key & key, const Float64 & default_value) const; -template void NamedCollection::set(const NamedCollection::Key & key, const String & value, bool update_if_exists); -template void NamedCollection::set(const NamedCollection::Key & key, const UInt64 & value, bool update_if_exists); -template void NamedCollection::set(const NamedCollection::Key & key, const Int64 & value, bool update_if_exists); -template void NamedCollection::set(const NamedCollection::Key & key, const Float64 & value, bool update_if_exists); +template void NamedCollection::set(const NamedCollection::Key & key, const String & value); +template void NamedCollection::set(const NamedCollection::Key & key, const String & value); +template void NamedCollection::set(const NamedCollection::Key & key, const UInt64 & value); +template void NamedCollection::set(const NamedCollection::Key & key, const UInt64 & value); +template void NamedCollection::set(const NamedCollection::Key & key, const Int64 & value); +template void NamedCollection::set(const NamedCollection::Key & key, const Int64 & value); +template void NamedCollection::set(const NamedCollection::Key & key, const Float64 & value); +template void NamedCollection::set(const NamedCollection::Key & key, const Float64 & value); + +template void NamedCollection::setOrUpdate(const NamedCollection::Key & key, const String & value); +template void NamedCollection::setOrUpdate(const NamedCollection::Key & key, const String & value); +template void NamedCollection::setOrUpdate(const NamedCollection::Key & key, const UInt64 & value); +template void NamedCollection::setOrUpdate(const NamedCollection::Key & key, const UInt64 & value); +template void NamedCollection::setOrUpdate(const NamedCollection::Key & key, const Int64 & value); +template void NamedCollection::setOrUpdate(const NamedCollection::Key & key, const Int64 & value); +template void NamedCollection::setOrUpdate(const NamedCollection::Key & key, const Float64 & value); +template void NamedCollection::setOrUpdate(const NamedCollection::Key & key, const Float64 & value); + +template void NamedCollection::remove(const Key & key); +template void NamedCollection::remove(const Key & key); } diff --git a/src/Storages/NamedCollections.h b/src/Storages/NamedCollections.h index 83bb1dd964e..f7181c2b539 100644 --- a/src/Storages/NamedCollections.h +++ b/src/Storages/NamedCollections.h @@ -1,15 +1,13 @@ #pragma once - #include -#include +#include +#include +namespace Poco { namespace Util { class AbstractConfiguration; } } namespace DB { -class NamedCollection; -using NamedCollectionPtr = std::shared_ptr; - /** * Class to represent arbitrary-structured named collection object. * It can be defined via config or via SQL command. @@ -22,40 +20,58 @@ using NamedCollectionPtr = std::shared_ptr; */ class NamedCollection { -private: - class Impl; - using ImplPtr = std::unique_ptr; - - ImplPtr pimpl; - public: using Key = std::string; using Keys = std::set; + using SourceId = NamedCollectionUtils::SourceId; - static NamedCollectionPtr create( - const Poco::Util::AbstractConfiguration & config, - const std::string & collection_name); - - NamedCollection( + static MutableNamedCollectionPtr create( const Poco::Util::AbstractConfiguration & config, + const std::string & collection_name, const std::string & collection_path, - const Keys & keys); - - explicit NamedCollection(ImplPtr pimpl_); + const Keys & keys, + SourceId source_id_, + bool is_mutable_); template T get(const Key & key) const; template T getOrDefault(const Key & key, const T & default_value) const; - template void set(const Key & key, const T & value, bool update_if_exists = false); + std::unique_lock lock(); - void remove(const Key & key); + template void set(const Key & key, const T & value); - std::shared_ptr duplicate() const; + template void setOrUpdate(const Key & key, const T & value); + + template void remove(const Key & key); + + MutableNamedCollectionPtr duplicate() const; Keys getKeys() const; std::string dumpStructure() const; + + bool isMutable() const { return is_mutable; } + + SourceId getSourceId() const { return source_id; } + +private: + class Impl; + using ImplPtr = std::unique_ptr; + + NamedCollection( + ImplPtr pimpl_, + const std::string & collection_name, + SourceId source_id, + bool is_mutable); + + void assertMutable() const; + + ImplPtr pimpl; + const std::string collection_name; + const SourceId source_id; + const bool is_mutable; + mutable std::mutex mutex; }; /** @@ -66,42 +82,51 @@ class NamedCollectionFactory : boost::noncopyable public: static NamedCollectionFactory & instance(); - void initialize(const Poco::Util::AbstractConfiguration & config_); - - void reload(const Poco::Util::AbstractConfiguration & config_); - bool exists(const std::string & collection_name) const; NamedCollectionPtr get(const std::string & collection_name) const; NamedCollectionPtr tryGet(const std::string & collection_name) const; - void add( - const std::string & collection_name, - NamedCollectionPtr collection); + MutableNamedCollectionPtr getMutable(const std::string & collection_name) const; + + void add(const std::string & collection_name, MutableNamedCollectionPtr collection); + + void add(NamedCollectionsMap collections); + + void update(NamedCollectionsMap collections); void remove(const std::string & collection_name); - using NamedCollections = std::unordered_map; - NamedCollections getAll() const; + void removeIfExists(const std::string & collection_name); + + void removeById(NamedCollectionUtils::SourceId id); + + NamedCollectionsMap getAll() const; private: - void assertInitialized(std::lock_guard & lock) const; - - NamedCollectionPtr getImpl( - const std::string & collection_name, - std::lock_guard & lock) const; - bool existsUnlocked( const std::string & collection_name, std::lock_guard & lock) const; - mutable NamedCollections loaded_named_collections; + MutableNamedCollectionPtr tryGetUnlocked( + const std::string & collection_name, + std::lock_guard & lock) const; - const Poco::Util::AbstractConfiguration * config; + void addUnlocked( + const std::string & collection_name, + MutableNamedCollectionPtr collection, + std::lock_guard & lock); + + bool removeIfExistsUnlocked( + const std::string & collection_name, + std::lock_guard & lock); + + mutable NamedCollectionsMap loaded_named_collections; - bool is_initialized = false; mutable std::mutex mutex; + bool is_initialized = false; }; + } diff --git a/src/Storages/NamedCollections_fwd.h b/src/Storages/NamedCollections_fwd.h new file mode 100644 index 00000000000..47ebe81c91f --- /dev/null +++ b/src/Storages/NamedCollections_fwd.h @@ -0,0 +1,12 @@ +#pragma once +#include + +namespace DB +{ + +class NamedCollection; +using NamedCollectionPtr = std::shared_ptr; +using MutableNamedCollectionPtr = std::shared_ptr; +using NamedCollectionsMap = std::map; + +} diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 57f5ddd86e6..bce3fee71f7 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -938,24 +938,24 @@ ProducerBufferPtr StorageRabbitMQ::createWriteBuffer() bool StorageRabbitMQ::checkDependencies(const StorageID & table_id) { // Check if all dependencies are attached - auto dependencies = DatabaseCatalog::instance().getDependencies(table_id); - if (dependencies.empty()) + auto view_ids = DatabaseCatalog::instance().getDependentViews(table_id); + if (view_ids.empty()) return true; // Check the dependencies are ready? - for (const auto & db_tab : dependencies) + for (const auto & view_id : view_ids) { - auto table = DatabaseCatalog::instance().tryGetTable(db_tab, getContext()); - if (!table) + auto view = DatabaseCatalog::instance().tryGetTable(view_id, getContext()); + if (!view) return false; // If it materialized view, check it's target table - auto * materialized_view = dynamic_cast(table.get()); + auto * materialized_view = dynamic_cast(view.get()); if (materialized_view && !materialized_view->tryGetTargetTable()) return false; // Check all its dependencies - if (!checkDependencies(db_tab)) + if (!checkDependencies(view_id)) return false; } @@ -984,10 +984,10 @@ void StorageRabbitMQ::streamingToViewsFunc() auto table_id = getStorageID(); // Check if at least one direct dependency is attached - size_t dependencies_count = DatabaseCatalog::instance().getDependencies(table_id).size(); + size_t num_views = DatabaseCatalog::instance().getDependentViews(table_id).size(); bool rabbit_connected = connection->isConnected() || connection->reconnect(); - if (dependencies_count && rabbit_connected) + if (num_views && rabbit_connected) { initializeBuffers(); auto start_time = std::chrono::steady_clock::now(); @@ -1000,7 +1000,7 @@ void StorageRabbitMQ::streamingToViewsFunc() if (!checkDependencies(table_id)) break; - LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count); + LOG_DEBUG(log, "Started streaming to {} attached views", num_views); if (streamToViews()) { diff --git a/src/Storages/RocksDB/EmbeddedRocksDBSink.cpp b/src/Storages/RocksDB/EmbeddedRocksDBSink.cpp index c39e70745fd..b1b158a2aa5 100644 --- a/src/Storages/RocksDB/EmbeddedRocksDBSink.cpp +++ b/src/Storages/RocksDB/EmbeddedRocksDBSink.cpp @@ -46,7 +46,7 @@ void EmbeddedRocksDBSink::consume(Chunk chunk) size_t idx = 0; for (const auto & elem : block) { - elem.type->getDefaultSerialization()->serializeBinary(*elem.column, i, idx == primary_key_pos ? wb_key : wb_value); + elem.type->getDefaultSerialization()->serializeBinary(*elem.column, i, idx == primary_key_pos ? wb_key : wb_value, {}); ++idx; } status = batch.Put(wb_key.str(), wb_value.str()); diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index 46ddb650eee..2fcedf550e8 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -259,7 +259,7 @@ void StorageEmbeddedRocksDB::mutate(const MutationCommands & commands, ContextPt { wb_key.restart(); - column_it->type->getDefaultSerialization()->serializeBinary(*column, i, wb_key); + column_it->type->getDefaultSerialization()->serializeBinary(*column, i, wb_key, {}); auto status = batch.Delete(wb_key.str()); if (!status.ok()) throw Exception("RocksDB write error: " + status.ToString(), ErrorCodes::ROCKSDB_ERROR); diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 65b4dce3ad2..e382e7f7bbb 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -434,7 +434,7 @@ void StorageBuffer::read( } -static void appendBlock(const Block & from, Block & to) +static void appendBlock(Poco::Logger * log, const Block & from, Block & to) { size_t rows = from.rows(); size_t old_rows = to.rows(); @@ -456,7 +456,24 @@ static void appendBlock(const Block & from, Block & to) for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) { const IColumn & col_from = *from.getByPosition(column_no).column.get(); - last_col = IColumn::mutate(std::move(to.getByPosition(column_no).column)); + { + /// Usually IColumn::mutate() here will simply move pointers, + /// however in case of parallel reading from it via SELECT, it + /// is possible for the full IColumn::clone() here, and in this + /// case it may fail due to MEMORY_LIMIT_EXCEEDED, and this + /// breaks the rollback, since the column got lost, it is + /// neither in last_col nor in "to" block. + /// + /// The safest option here, is to do a full clone every time, + /// however, it is overhead. And it looks like the only + /// exception that is possible here is MEMORY_LIMIT_EXCEEDED, + /// and it is better to simply suppress it, to avoid overhead + /// for every INSERT into Buffer (Anyway we have a + /// LOGICAL_ERROR in rollback that will bail if something else + /// will happens here). + LockMemoryExceptionInThread temporarily_ignore_any_memory_limits(VariableContext::Global); + last_col = IColumn::mutate(std::move(to.getByPosition(column_no).column)); + } /// In case of ColumnAggregateFunction aggregate states will /// be allocated from the query context but can be destroyed from the @@ -468,7 +485,10 @@ static void appendBlock(const Block & from, Block & to) last_col->ensureOwnership(); last_col->insertRangeFrom(col_from, 0, rows); - to.getByPosition(column_no).column = std::move(last_col); + { + DENY_ALLOCATIONS_IN_SCOPE; + to.getByPosition(column_no).column = std::move(last_col); + } } CurrentMetrics::add(CurrentMetrics::StorageBufferRows, rows); CurrentMetrics::add(CurrentMetrics::StorageBufferBytes, to.bytes() - old_bytes); @@ -481,6 +501,9 @@ static void appendBlock(const Block & from, Block & to) /// So ignore any memory limits, even global (since memory tracking has drift). LockMemoryExceptionInThread temporarily_ignore_any_memory_limits(VariableContext::Global); + /// But first log exception to get more details in case of LOGICAL_ERROR + tryLogCurrentException(log, "Caught exception while adding data to buffer, rolling back..."); + try { for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) @@ -625,7 +648,7 @@ private: size_t old_rows = buffer.data.rows(); size_t old_bytes = buffer.data.allocatedBytes(); - appendBlock(sorted_block, buffer.data); + appendBlock(storage.log, sorted_block, buffer.data); storage.total_writes.rows += (buffer.data.rows() - old_rows); storage.total_writes.bytes += (buffer.data.allocatedBytes() - old_bytes); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index f7f68eba30f..51cca15e5ab 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -703,7 +703,7 @@ void StorageDistributed::read( select_stream_factory, modified_query_ast, local_context, query_info, sharding_key_expr, sharding_key_column_name, - query_info.cluster); + query_info.cluster, processed_stage); else ClusterProxy::executeQuery( query_plan, header, processed_stage, diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 21be205c0f6..bd255a952dc 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -111,7 +111,7 @@ public: size_t idx = 0; for (const auto & elem : block) { - elem.type->getDefaultSerialization()->serializeBinary(*elem.column, i, idx == primary_key_pos ? wb_key : wb_value); + elem.type->getDefaultSerialization()->serializeBinary(*elem.column, i, idx == primary_key_pos ? wb_key : wb_value, {}); ++idx; } diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index e256e087728..ed01ca9cec4 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -210,7 +210,7 @@ void StorageMaterializedView::drop() auto table_id = getStorageID(); const auto & select_query = getInMemoryMetadataPtr()->getSelectQuery(); if (!select_query.select_table_id.empty()) - DatabaseCatalog::instance().removeDependency(select_query.select_table_id, table_id); + DatabaseCatalog::instance().removeViewDependency(select_query.select_table_id, table_id); dropInnerTableIfAny(true, getContext()); } @@ -266,7 +266,7 @@ void StorageMaterializedView::alter( const auto & new_select = new_metadata.select; const auto & old_select = old_metadata.getSelectQuery(); - DatabaseCatalog::instance().updateDependency(old_select.select_table_id, table_id, new_select.select_table_id, table_id); + DatabaseCatalog::instance().updateViewDependency(old_select.select_table_id, table_id, new_select.select_table_id, table_id); new_metadata.setSelectQuery(new_select); } @@ -364,7 +364,7 @@ void StorageMaterializedView::renameInMemory(const StorageID & new_table_id) } const auto & select_query = metadata_snapshot->getSelectQuery(); // TODO Actually we don't need to update dependency if MV has UUID, but then db and table name will be outdated - DatabaseCatalog::instance().updateDependency(select_query.select_table_id, old_table_id, select_query.select_table_id, getStorageID()); + DatabaseCatalog::instance().updateViewDependency(select_query.select_table_id, old_table_id, select_query.select_table_id, getStorageID()); } void StorageMaterializedView::startup() @@ -372,7 +372,7 @@ void StorageMaterializedView::startup() auto metadata_snapshot = getInMemoryMetadataPtr(); const auto & select_query = metadata_snapshot->getSelectQuery(); if (!select_query.select_table_id.empty()) - DatabaseCatalog::instance().addDependency(select_query.select_table_id, getStorageID()); + DatabaseCatalog::instance().addViewDependency(select_query.select_table_id, getStorageID()); } void StorageMaterializedView::shutdown() @@ -381,7 +381,7 @@ void StorageMaterializedView::shutdown() const auto & select_query = metadata_snapshot->getSelectQuery(); /// Make sure the dependency is removed after DETACH TABLE if (!select_query.select_table_id.empty()) - DatabaseCatalog::instance().removeDependency(select_query.select_table_id, getStorageID()); + DatabaseCatalog::instance().removeViewDependency(select_query.select_table_id, getStorageID()); } StoragePtr StorageMaterializedView::getTargetTable() const diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 24719bfb1d6..b83b4e97ebe 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -279,25 +279,6 @@ void StorageMergeTree::drop() dropAllData(); } -void StorageMergeTree::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &) -{ - { - /// Asks to complete merges and does not allow them to start. - /// This protects against "revival" of data for a removed partition after completion of merge. - auto merge_blocker = stopMergesAndWait(); - - auto data_parts_lock = lockParts(); - auto parts_to_remove = getVisibleDataPartsVectorUnlocked(local_context, data_parts_lock); - removePartsFromWorkingSet(local_context->getCurrentTransaction().get(), parts_to_remove, true, data_parts_lock); - - LOG_INFO(log, "Removed {} parts.", parts_to_remove.size()); - } - - clearOldMutations(true); - clearOldPartsFromFilesystem(); -} - - void StorageMergeTree::alter( const AlterCommands & commands, ContextPtr local_context, @@ -826,22 +807,28 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge( CurrentlyMergingPartsTaggerPtr merging_tagger; MergeList::EntryPtr merge_entry; - auto can_merge = [this, &lock](const DataPartPtr & left, const DataPartPtr & right, const MergeTreeTransaction * tx, String *) -> bool + auto can_merge = [this, &lock](const DataPartPtr & left, const DataPartPtr & right, const MergeTreeTransaction * tx, String * disable_reason) -> bool { if (tx) { /// Cannot merge parts if some of them are not visible in current snapshot /// TODO Transactions: We can use simplified visibility rules (without CSN lookup) here - if (left && !left->version.isVisible(tx->getSnapshot(), Tx::EmptyTID)) - return false; - if (right && !right->version.isVisible(tx->getSnapshot(), Tx::EmptyTID)) + if ((left && !left->version.isVisible(tx->getSnapshot(), Tx::EmptyTID)) + || (right && !right->version.isVisible(tx->getSnapshot(), Tx::EmptyTID))) + { + if (disable_reason) + *disable_reason = "Some part is not visible in transaction"; return false; + } /// Do not try to merge parts that are locked for removal (merge will probably fail) - if (left && left->version.isRemovalTIDLocked()) - return false; - if (right && right->version.isRemovalTIDLocked()) + if ((left && left->version.isRemovalTIDLocked()) + || (right && right->version.isRemovalTIDLocked())) + { + if (disable_reason) + *disable_reason = "Some part is locked for removal in another cuncurrent transaction"; return false; + } } /// This predicate is checked for the first part of each range. @@ -1398,7 +1385,6 @@ ActionLock StorageMergeTree::stopMergesAndWait() return merge_blocker; } - MergeTreeDataPartPtr StorageMergeTree::outdatePart(MergeTreeTransaction * txn, const String & part_name, bool force) { if (force) @@ -1407,7 +1393,8 @@ MergeTreeDataPartPtr StorageMergeTree::outdatePart(MergeTreeTransaction * txn, c auto merge_blocker = stopMergesAndWait(); auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Active}); if (!part) - throw Exception("Part " + part_name + " not found, won't try to drop it.", ErrorCodes::NO_SUCH_DATA_PART); + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "Part {} not found, won't try to drop it.", part_name); + removePartsFromWorkingSet(txn, {part}, true); return part; } @@ -1434,72 +1421,261 @@ MergeTreeDataPartPtr StorageMergeTree::outdatePart(MergeTreeTransaction * txn, c void StorageMergeTree::dropPartNoWaitNoThrow(const String & part_name) { if (auto part = outdatePart(NO_TRANSACTION_RAW, part_name, /*force=*/ false)) - dropPartsImpl({part}, /*detach=*/ false); + { + if (deduplication_log) + { + deduplication_log->dropPart(part->info); + } + + /// Need to destroy part objects before clearing them from filesystem. + part.reset(); + + clearOldPartsFromFilesystem(); + + LOG_INFO(log, "Removed 1 part {}.", part_name); + } /// Else nothing to do, part was removed in some different way } -void StorageMergeTree::dropPart(const String & part_name, bool detach, ContextPtr query_context) +struct FutureNewEmptyPart { - if (auto part = outdatePart(query_context->getCurrentTransaction().get(), part_name, /*force=*/ true)) - dropPartsImpl({part}, detach); + MergeTreePartInfo part_info; + MergeTreePartition partition; + std::string part_name; + + scope_guard tmp_dir_guard; + + StorageMergeTree::MutableDataPartPtr data_part; + + std::string getDirName() const { return StorageMergeTree::EMPTY_PART_TMP_PREFIX + part_name; } +}; + +using FutureNewEmptyParts = std::vector; + +Strings getPartsNames(const FutureNewEmptyParts & parts) +{ + Strings part_names; + for (const auto & p : parts) + part_names.push_back(p.part_name); + return part_names; } -void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, ContextPtr local_context) +FutureNewEmptyParts initCoverageWithNewEmptyParts(const DataPartsVector & old_parts) { - DataPartsVector parts_to_remove; - /// New scope controls lifetime of merge_blocker. + FutureNewEmptyParts future_parts; + + for (const auto & old_part : old_parts) { - /// Asks to complete merges and does not allow them to start. - /// This protects against "revival" of data for a removed partition after completion of merge. - auto merge_blocker = stopMergesAndWait(); - auto data_parts_lock = lockParts(); - const auto * partition_ast = partition->as(); - if (partition_ast && partition_ast->all) - parts_to_remove = getVisibleDataPartsVectorUnlocked(local_context, data_parts_lock); - else - { - String partition_id = getPartitionIDFromQuery(partition, local_context, &data_parts_lock); - parts_to_remove = getVisibleDataPartsVectorInPartition(local_context, partition_id, data_parts_lock); - } - /// TODO should we throw an exception if parts_to_remove is empty? - removePartsFromWorkingSet(local_context->getCurrentTransaction().get(), parts_to_remove, true, data_parts_lock); + future_parts.emplace_back(); + auto & new_part = future_parts.back(); + + new_part.part_info = old_part->info; + new_part.part_info.level += 1; + new_part.partition = old_part->partition; + new_part.part_name = old_part->getNewName(new_part.part_info); } - dropPartsImpl(std::move(parts_to_remove), detach); + return future_parts; } -void StorageMergeTree::dropPartsImpl(DataPartsVector && parts_to_remove, bool detach) +StorageMergeTree::MutableDataPartsVector createEmptyDataParts(MergeTreeData & data, FutureNewEmptyParts & future_parts, const MergeTreeTransactionPtr & txn) { - auto metadata_snapshot = getInMemoryMetadataPtr(); + StorageMergeTree::MutableDataPartsVector data_parts; + for (auto & part: future_parts) + data_parts.push_back(data.createEmptyPart(part.part_info, part.partition, part.part_name, txn)); + return data_parts; +} - if (detach) +void captureTmpDirectoryHolders(MergeTreeData & data, FutureNewEmptyParts & future_parts) +{ + for (auto & part : future_parts) + part.tmp_dir_guard = data.getTemporaryPartDirectoryHolder(part.getDirName()); +} + +void StorageMergeTree::renameAndCommitEmptyParts(MutableDataPartsVector & new_parts, Transaction & transaction) +{ + DataPartsVector covered_parts; + + for (auto & part: new_parts) { - /// If DETACH clone parts to detached/ directory - /// NOTE: no race with background cleanup until we hold pointers to parts - for (const auto & part : parts_to_remove) + DataPartsVector covered_parts_by_one_part = renameTempPartAndReplace(part, transaction); + + if (covered_parts_by_one_part.size() > 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} expected to cover not more then 1 part. {} covered parts have been found. This is a bug.", + part->name, covered_parts_by_one_part.size()); + + std::move(covered_parts_by_one_part.begin(), covered_parts_by_one_part.end(), std::back_inserter(covered_parts)); + } + + LOG_INFO(log, "Remove {} parts by covering them with empty {} parts. With txn {}.", + covered_parts.size(), new_parts.size(), transaction.getTID()); + + transaction.commit(); + + /// Remove covered parts without waiting for old_parts_lifetime seconds. + for (auto & part: covered_parts) + part->remove_time.store(0, std::memory_order_relaxed); + + if (deduplication_log) + for (const auto & part : covered_parts) + deduplication_log->dropPart(part->info); +} + +void StorageMergeTree::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr query_context, TableExclusiveLockHolder &) +{ + /// Asks to complete merges and does not allow them to start. + /// This protects against "revival" of data for a removed partition after completion of merge. + auto merge_blocker = stopMergesAndWait(); + + Stopwatch watch; + + auto txn = query_context->getCurrentTransaction(); + MergeTreeData::Transaction transaction(*this, txn.get()); + { + auto operation_data_parts_lock = lockOperationsWithParts(); + + auto parts = getVisibleDataPartsVector(query_context); + + auto future_parts = initCoverageWithNewEmptyParts(parts); + + LOG_TEST(log, "Made {} empty parts in order to cover {} parts. Empty parts: {}, covered parts: {}. With txn {}", + future_parts.size(), parts.size(), + fmt::join(getPartsNames(future_parts), ", "), fmt::join(getPartsNames(parts), ", "), + transaction.getTID()); + + captureTmpDirectoryHolders(*this, future_parts); + + auto new_data_parts = createEmptyDataParts(*this, future_parts, txn); + renameAndCommitEmptyParts(new_data_parts, transaction); + + PartLog::addNewParts(query_context, new_data_parts, watch.elapsed()); + + LOG_INFO(log, "Truncated table with {} parts by replacing them with new empty {} parts. With txn {}", + parts.size(), future_parts.size(), + transaction.getTID()); + } + + /// Old parts are needed to be destroyed before clearing them from filesystem. + clearOldMutations(true); + clearOldPartsFromFilesystem(); + clearEmptyParts(); +} + +void StorageMergeTree::dropPart(const String & part_name, bool detach, ContextPtr query_context) +{ + /// Asks to complete merges and does not allow them to start. + /// This protects against "revival" of data for a removed partition after completion of merge. + auto merge_blocker = stopMergesAndWait(); + + Stopwatch watch; + + /// It's important to create it outside of lock scope because + /// otherwise it can lock parts in destructor and deadlock is possible. + auto txn = query_context->getCurrentTransaction(); + MergeTreeData::Transaction transaction(*this, txn.get()); + { + auto operation_data_parts_lock = lockOperationsWithParts(); + + auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Active}); + if (!part) + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "Part {} not found, won't try to drop it.", part_name); + + if (detach) { + auto metadata_snapshot = getInMemoryMetadataPtr(); LOG_INFO(log, "Detaching {}", part->getDataPartStorage().getPartDirectory()); part->makeCloneInDetached("", metadata_snapshot); } + + { + auto future_parts = initCoverageWithNewEmptyParts({part}); + + LOG_TEST(log, "Made {} empty parts in order to cover {} part. With txn {}", + fmt::join(getPartsNames(future_parts), ", "), fmt::join(getPartsNames({part}), ", "), + transaction.getTID()); + + captureTmpDirectoryHolders(*this, future_parts); + + auto new_data_parts = createEmptyDataParts(*this, future_parts, txn); + renameAndCommitEmptyParts(new_data_parts, transaction); + + PartLog::addNewParts(query_context, new_data_parts, watch.elapsed()); + + const auto * op = detach ? "Detached" : "Dropped"; + LOG_INFO(log, "{} {} part by replacing it with new empty {} part. With txn {}", + op, part->name, future_parts[0].part_name, + transaction.getTID()); + } } - if (deduplication_log) - { - for (const auto & part : parts_to_remove) - deduplication_log->dropPart(part->info); - } - - if (detach) - LOG_INFO(log, "Detached {} parts.", parts_to_remove.size()); - else - LOG_INFO(log, "Removed {} parts.", parts_to_remove.size()); - - /// Need to destroy part objects before clearing them from filesystem. - parts_to_remove.clear(); + /// Old part objects is needed to be destroyed before clearing them from filesystem. + clearOldMutations(true); clearOldPartsFromFilesystem(); + clearEmptyParts(); } +void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, ContextPtr query_context) +{ + const auto * partition_ast = partition->as(); + + /// Asks to complete merges and does not allow them to start. + /// This protects against "revival" of data for a removed partition after completion of merge. + auto merge_blocker = stopMergesAndWait(); + + Stopwatch watch; + + /// It's important to create it outside of lock scope because + /// otherwise it can lock parts in destructor and deadlock is possible. + auto txn = query_context->getCurrentTransaction(); + MergeTreeData::Transaction transaction(*this, txn.get()); + { + auto operation_data_parts_lock = lockOperationsWithParts(); + + DataPartsVector parts; + { + if (partition_ast && partition_ast->all) + parts = getVisibleDataPartsVector(query_context); + else + { + String partition_id = getPartitionIDFromQuery(partition, query_context); + parts = getVisibleDataPartsVectorInPartition(query_context, partition_id); + } + } + + if (detach) + for (const auto & part : parts) + { + auto metadata_snapshot = getInMemoryMetadataPtr(); + LOG_INFO(log, "Detaching {}", part->getDataPartStorage().getPartDirectory()); + part->makeCloneInDetached("", metadata_snapshot); + } + + auto future_parts = initCoverageWithNewEmptyParts(parts); + + LOG_TEST(log, "Made {} empty parts in order to cover {} parts. Empty parts: {}, covered parts: {}. With txn {}", + future_parts.size(), parts.size(), + fmt::join(getPartsNames(future_parts), ", "), fmt::join(getPartsNames(parts), ", "), + transaction.getTID()); + + captureTmpDirectoryHolders(*this, future_parts); + + auto new_data_parts = createEmptyDataParts(*this, future_parts, txn); + renameAndCommitEmptyParts(new_data_parts, transaction); + + PartLog::addNewParts(query_context, new_data_parts, watch.elapsed()); + + const auto * op = detach ? "Detached" : "Dropped"; + LOG_INFO(log, "{} partition with {} parts by replacing them with new empty {} parts. With txn {}", + op, parts.size(), future_parts.size(), + transaction.getTID()); + } + + /// Old parts are needed to be destroyed before clearing them from filesystem. + clearOldMutations(true); + clearOldPartsFromFilesystem(); + clearEmptyParts(); +} PartitionCommandsResultInfo StorageMergeTree::attachPartition( const ASTPtr & partition, const StorageMetadataPtr & /* metadata_snapshot */, diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 745546b96f6..11d7d8f8fc1 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -169,6 +169,8 @@ private: String * out_disable_reason = nullptr, bool optimize_skip_merged_partitions = false); + void renameAndCommitEmptyParts(MutableDataPartsVector & new_parts, Transaction & transaction); + /// Make part state outdated and queue it to remove without timeout /// If force, then stop merges and block them until part state became outdated. Throw exception if part doesn't exists /// If not force, then take merges selector and check that part is not participating in background operations. @@ -217,7 +219,6 @@ private: void dropPartNoWaitNoThrow(const String & part_name) override; void dropPart(const String & part_name, bool detach, ContextPtr context) override; void dropPartition(const ASTPtr & partition, bool detach, ContextPtr context) override; - void dropPartsImpl(DataPartsVector && parts_to_remove, bool detach); PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, ContextPtr context) override; void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr context) override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9eacec2351e..b6e7864ac80 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -67,7 +67,6 @@ #include #include #include -#include #include #include @@ -131,7 +130,7 @@ namespace ErrorCodes extern const int NO_ZOOKEEPER; extern const int INCORRECT_DATA; extern const int INCOMPATIBLE_COLUMNS; - extern const int REPLICA_IS_ALREADY_EXIST; + extern const int REPLICA_ALREADY_EXISTS; extern const int NO_REPLICA_HAS_PART; extern const int LOGICAL_ERROR; extern const int TOO_MANY_UNEXPECTED_DATA_PARTS; @@ -779,7 +778,7 @@ bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr /// Do not use LOGICAL_ERROR code, because it may happen if user has specified wrong zookeeper_path throw Exception("Cannot create table, because it is created concurrently every time " "or because of wrong zookeeper_path " - "or because of logical error", ErrorCodes::REPLICA_IS_ALREADY_EXIST); + "or because of logical error", ErrorCodes::REPLICA_ALREADY_EXISTS); } void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot) @@ -843,7 +842,7 @@ void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metada switch (code) { case Coordination::Error::ZNODEEXISTS: - throw Exception(ErrorCodes::REPLICA_IS_ALREADY_EXIST, "Replica {} already exists", replica_path); + throw Exception(ErrorCodes::REPLICA_ALREADY_EXISTS, "Replica {} already exists", replica_path); case Coordination::Error::ZBADVERSION: LOG_ERROR(log, "Retrying createReplica(), because some other replicas were created at the same time"); break; @@ -1554,7 +1553,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) if (entry.type == LogEntry::ATTACH_PART) { - if (MutableDataPartPtr part = attachPartHelperFoundValidPart(entry); part) + if (MutableDataPartPtr part = attachPartHelperFoundValidPart(entry)) { LOG_TRACE(log, "Found valid local part for {}, preparing the transaction", part->name); @@ -3129,7 +3128,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() auto zookeeper = getZooKeeperAndAssertNotReadonly(); - ReplicatedMergeTreeMergePredicate merge_pred = queue.getMergePredicate(zookeeper); + ReplicatedMergeTreeMergePredicate merge_pred = queue.getMergePredicate(zookeeper, getAllPartitionIds()); /// If many merges is already queued, then will queue only small enough merges. /// Otherwise merge queue could be filled with only large merges, @@ -4553,10 +4552,11 @@ bool StorageReplicatedMergeTree::optimize( if (!is_leader) throw Exception("OPTIMIZE cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); - auto handle_noop = [&] (const String & message) + auto handle_noop = [&] (const char * fmt_string, auto ...args) { + LOG_DEBUG(log, fmt::runtime(fmt_string), args...); if (query_context->getSettingsRef().optimize_throw_if_noop) - throw Exception(message, ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); + throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, fmt::runtime(fmt_string), args...); return false; }; @@ -4574,7 +4574,19 @@ bool StorageReplicatedMergeTree::optimize( /// We must select parts for merge under merge_selecting_mutex because other threads /// (merge_selecting_thread or OPTIMIZE queries) could assign new merges. std::lock_guard merge_selecting_lock(merge_selecting_mutex); - ReplicatedMergeTreeMergePredicate can_merge = queue.getMergePredicate(zookeeper); + PartitionIdsHint partition_ids_hint; + if (partition_id.empty()) + { + partition_ids_hint = getAllPartitionIds(); + } + else + { + auto parts_lock = lockParts(); + if (!getAnyPartInPartition(partition_id, parts_lock)) + handle_noop("Cannot select parts for optimization: there are no parts in partition {}", partition_id); + partition_ids_hint.insert(partition_id); + } + ReplicatedMergeTreeMergePredicate can_merge = queue.getMergePredicate(zookeeper, std::move(partition_ids_hint)); auto future_merged_part = std::make_shared(); if (storage_settings.get()->assign_part_uuids) @@ -4607,9 +4619,7 @@ bool StorageReplicatedMergeTree::optimize( assert(disable_reason != unknown_disable_reason); if (!partition_id.empty()) disable_reason += fmt::format(" (in partition {})", partition_id); - String message = fmt::format(message_fmt, disable_reason); - LOG_INFO(log, fmt::runtime(message)); - return handle_noop(message); + return handle_noop(message_fmt, disable_reason); } ReplicatedMergeTreeLogEntryData merge_entry; @@ -4621,9 +4631,8 @@ bool StorageReplicatedMergeTree::optimize( if (create_result == CreateMergeEntryResult::MissingPart) { - String message = "Can't create merge queue node in ZooKeeper, because some parts are missing"; - LOG_TRACE(log, fmt::runtime(message)); - return handle_noop(message); + static constexpr const char * message_fmt = "Can't create merge queue node in ZooKeeper, because some parts are missing"; + return handle_noop(message_fmt); } if (create_result == CreateMergeEntryResult::LogUpdated) @@ -4634,9 +4643,8 @@ bool StorageReplicatedMergeTree::optimize( } assert(try_no == max_retries); - String message = fmt::format("Can't create merge queue node in ZooKeeper, because log was updated in every of {} tries", try_no); - LOG_TRACE(log, fmt::runtime(message)); - return handle_noop(message); + static constexpr const char * message_fmt = "Can't create merge queue node in ZooKeeper, because log was updated in every of {} tries"; + return handle_noop(message_fmt, try_no); }; bool assigned = false; @@ -7058,7 +7066,7 @@ void StorageReplicatedMergeTree::movePartitionToShard( throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Part {} does not have an uuid assigned and it can't be moved between shards", part_name); - ReplicatedMergeTreeMergePredicate merge_pred = queue.getMergePredicate(zookeeper); + ReplicatedMergeTreeMergePredicate merge_pred = queue.getMergePredicate(zookeeper, PartitionIdsHint{part_info.partition_id}); /// The following block is pretty much copy & paste from StorageReplicatedMergeTree::dropPart to avoid conflicts while this is WIP. /// Extract it to a common method and re-use it before merging. @@ -7266,7 +7274,7 @@ bool StorageReplicatedMergeTree::dropPartImpl( while (true) { - ReplicatedMergeTreeMergePredicate merge_pred = queue.getMergePredicate(zookeeper); + ReplicatedMergeTreeMergePredicate merge_pred = queue.getMergePredicate(zookeeper, PartitionIdsHint{part_info.partition_id}); auto part = getPartIfExists(part_info, {MergeTreeDataPartState::Active}); @@ -7645,7 +7653,15 @@ void StorageReplicatedMergeTree::createTableSharedID() const return; } - auto zookeeper = getZooKeeper(); + /// We may call getTableSharedID when table is shut down. If exception happen, restarting thread will be already turned + /// off and nobody will reconnect our zookeeper connection. In this case we use zookeeper connection from + /// context. + ZooKeeperPtr zookeeper; + if (shutdown_called.load()) + zookeeper = getZooKeeperIfTableShutDown(); + else + zookeeper = getZooKeeper(); + String zookeeper_table_id_path = fs::path(zookeeper_path) / "table_shared_id"; String id; if (!zookeeper->tryGet(zookeeper_table_id_path, id)) @@ -8265,56 +8281,25 @@ bool StorageReplicatedMergeTree::checkIfDetachedPartitionExists(const String & p bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperPtr zookeeper, const String & lost_part_name) { LOG_INFO(log, "Going to replace lost part {} with empty part", lost_part_name); - auto metadata_snapshot = getInMemoryMetadataPtr(); - auto settings = getSettings(); - - constexpr static auto TMP_PREFIX = "tmp_empty_"; auto new_part_info = MergeTreePartInfo::fromPartName(lost_part_name, format_version); - auto block = metadata_snapshot->getSampleBlock(); - DB::IMergeTreeDataPart::TTLInfos move_ttl_infos; - - NamesAndTypesList columns = metadata_snapshot->getColumns().getAllPhysical().filter(block.getNames()); - ReservationPtr reservation = reserveSpacePreferringTTLRules(metadata_snapshot, 0, move_ttl_infos, time(nullptr), 0, true); - VolumePtr volume = getStoragePolicy()->getVolume(0); - - auto minmax_idx = std::make_shared(); - minmax_idx->update(block, getMinMaxColumnsNames(metadata_snapshot->getPartitionKey())); - - auto new_volume = createVolumeFromReservation(reservation, volume); - - auto data_part_storage = std::make_shared( - new_volume, - relative_data_path, - TMP_PREFIX + lost_part_name); - - data_part_storage->beginTransaction(); - - auto new_data_part = createPart( - lost_part_name, - choosePartType(0, block.rows()), - new_part_info, - data_part_storage); - - if (settings->assign_part_uuids) - new_data_part->uuid = UUIDHelpers::generateV4(); - - new_data_part->setColumns(columns, {}); - new_data_part->rows_count = block.rows(); + auto metadata_snapshot = getInMemoryMetadataPtr(); + MergeTreePartition partition; { - auto lock = lockParts(); + DataPartsLock lock = lockParts(); + auto parts_in_partition = getDataPartsPartitionRange(new_part_info.partition_id); if (!parts_in_partition.empty()) { - new_data_part->partition = (*parts_in_partition.begin())->partition; + partition = (*parts_in_partition.begin())->partition; } else if (auto parsed_partition = MergeTreePartition::tryParseValueFromID( new_part_info.partition_id, metadata_snapshot->getPartitionKey().sample_block)) { - new_data_part->partition = MergeTreePartition(*parsed_partition); + partition = MergeTreePartition(*parsed_partition); } else { @@ -8322,43 +8307,10 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP "resolve this manually using DROP/DETACH PARTITION.", lost_part_name, new_part_info.partition_id); return false; } - } - new_data_part->minmax_idx = std::move(minmax_idx); - new_data_part->is_temp = true; - - SyncGuardPtr sync_guard; - if (new_data_part->isStoredOnDisk()) - { - /// The name could be non-unique in case of stale files from previous runs. - if (data_part_storage->exists()) - { - LOG_WARNING(log, "Removing old temporary directory {}", new_data_part->getDataPartStorage().getFullPath()); - data_part_storage->removeRecursive(); - } - - data_part_storage->createDirectories(); - - if (getSettings()->fsync_part_directory) - sync_guard = data_part_storage->getDirectorySyncGuard(); - } - - /// This effectively chooses minimal compression method: - /// either default lz4 or compression method with zero thresholds on absolute and relative part size. - auto compression_codec = getContext()->chooseCompressionCodec(0, 0); - - const auto & index_factory = MergeTreeIndexFactory::instance(); - MergedBlockOutputStream out(new_data_part, metadata_snapshot, columns, - index_factory.getMany(metadata_snapshot->getSecondaryIndices()), compression_codec, NO_TRANSACTION_PTR); - - bool sync_on_insert = settings->fsync_after_insert; - - out.write(block); - /// TODO(ab): What projections should we add to the empty part? How can we make sure that it - /// won't block future merges? Perhaps we should also check part emptiness when selecting parts - /// to merge. - out.finalizePart(new_data_part, sync_on_insert); + MergeTreeData::MutableDataPartPtr new_data_part = createEmptyPart(new_part_info, partition, lost_part_name, NO_TRANSACTION_PTR); + new_data_part->name = lost_part_name; try { @@ -8391,7 +8343,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP /// We can enqueue part for check from DataPartExchange or SelectProcessor /// and it's hard to synchronize it with ReplicatedMergeTreeQueue and PartCheckThread... /// But at least we can ignore parts that are definitely not needed according to virtual parts and drop ranges. - auto pred = queue.getMergePredicate(zookeeper); + auto pred = queue.getMergePredicate(zookeeper, PartitionIdsHint{new_part_info.partition_id}); String covering_virtual = pred.getCoveringVirtualPart(lost_part_name); if (covering_virtual.empty()) { diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index a9602f38bff..6ea6bd129f6 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -832,6 +832,11 @@ std::shared_ptr StorageS3::createFileIterator( } } +bool StorageS3::supportsSubcolumns() const +{ + return FormatFactory::instance().checkIfFormatSupportsSubcolumns(format_name); +} + bool StorageS3::supportsSubsetOfColumns() const { return FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(format_name); diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index aa558ddc0de..2add41d4f95 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -254,6 +254,8 @@ private: ContextPtr ctx, std::unordered_map * object_infos = nullptr); + bool supportsSubcolumns() const override; + bool supportsSubsetOfColumns() const override; static std::optional tryGetColumnsFromCache( diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index ec970654b6e..b10f3c65ebf 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -103,8 +105,7 @@ Pipe StorageS3Cluster::read( auto callback = std::make_shared([iterator]() mutable -> String { return iterator->next(); }); /// Calculate the header. This is significant, because some columns could be thrown away in some cases like query with count(*) - Block header = - InterpreterSelectQuery(query_info.query, context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); + auto interpreter = InterpreterSelectQuery(query_info.query, context, SelectQueryOptions(processed_stage).analyze()); const Scalars & scalars = context->hasQueryContext() ? context->getQueryContext()->getScalars() : Scalars{}; @@ -112,11 +113,21 @@ Pipe StorageS3Cluster::read( const bool add_agg_info = processed_stage == QueryProcessingStage::WithMergeableState; - ASTPtr query_to_send = query_info.original_query->clone(); + ASTPtr query_to_send = interpreter.getQueryInfo().query->clone(); if (add_columns_structure_to_query) addColumnsStructureToQueryWithClusterEngine( query_to_send, StorageDictionary::generateNamesAndTypesDescription(storage_snapshot->metadata->getColumns().getAll()), 5, getName()); + RestoreQualifiedNamesVisitor::Data data; + data.distributed_table = DatabaseAndTableWithAlias(*getTableExpression(query_info.query->as(), 0)); + data.remote_table.database = context->getCurrentDatabase(); + data.remote_table.table = getName(); + RestoreQualifiedNamesVisitor(data).visit(query_to_send); + AddDefaultDatabaseVisitor visitor(context, context->getCurrentDatabase(), + /* only_replace_current_database_function_= */false, + /* only_replace_in_join_= */true); + visitor.visit(query_to_send); + const auto & current_settings = context->getSettingsRef(); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings); for (const auto & shard_info : cluster->getShardsInfo()) @@ -128,7 +139,7 @@ Pipe StorageS3Cluster::read( shard_info.pool, std::vector{try_result}, queryToString(query_to_send), - header, + interpreter.getSampleBlock(), context, /*throttler=*/nullptr, scalars, diff --git a/src/Storages/System/StorageSystemAsynchronousInserts.cpp b/src/Storages/System/StorageSystemAsynchronousInserts.cpp index 5ebdb828c34..15258ccfd7f 100644 --- a/src/Storages/System/StorageSystemAsynchronousInserts.cpp +++ b/src/Storages/System/StorageSystemAsynchronousInserts.cpp @@ -27,8 +27,6 @@ NamesAndTypesList StorageSystemAsynchronousInserts::getNamesAndTypes() {"total_bytes", std::make_shared()}, {"entries.query_id", std::make_shared(std::make_shared())}, {"entries.bytes", std::make_shared(std::make_shared())}, - {"entries.finished", std::make_shared(std::make_shared())}, - {"entries.exception", std::make_shared(std::make_shared())}, }; } @@ -40,78 +38,56 @@ void StorageSystemAsynchronousInserts::fillData(MutableColumns & res_columns, Co if (!insert_queue) return; - auto [queue, queue_lock] = insert_queue->getQueueLocked(); - for (const auto & [key, elem] : queue) + for (size_t shard_num = 0; shard_num < insert_queue->getPoolSize(); ++shard_num) { - std::lock_guard elem_lock(elem->mutex); + auto [queue, queue_lock] = insert_queue->getQueueLocked(shard_num); - if (!elem->data) - continue; - - auto time_in_microseconds = [](const time_point & timestamp) + for (const auto & [first_update, elem] : queue) { - auto time_diff = duration_cast(steady_clock::now() - timestamp); - auto time_us = (system_clock::now() - time_diff).time_since_epoch().count(); + const auto & [key, data] = elem; - DecimalUtils::DecimalComponents components{time_us / 1'000'000, time_us % 1'000'000}; - return DecimalField(DecimalUtils::decimalFromComponents(components, TIME_SCALE), TIME_SCALE); - }; - - const auto & insert_query = key.query->as(); - size_t i = 0; - - res_columns[i++]->insert(queryToString(insert_query)); - - /// If query is "INSERT INTO FUNCTION" then table_id is empty. - if (insert_query.table_id) - { - res_columns[i++]->insert(insert_query.table_id.getDatabaseName()); - res_columns[i++]->insert(insert_query.table_id.getTableName()); - } - else - { - res_columns[i++]->insertDefault(); - res_columns[i++]->insertDefault(); - } - - res_columns[i++]->insert(insert_query.format); - res_columns[i++]->insert(time_in_microseconds(elem->data->first_update)); - res_columns[i++]->insert(elem->data->size); - - Array arr_query_id; - Array arr_bytes; - Array arr_finished; - Array arr_exception; - - for (const auto & entry : elem->data->entries) - { - arr_query_id.push_back(entry->query_id); - arr_bytes.push_back(entry->bytes.size()); - arr_finished.push_back(entry->isFinished()); - - if (auto exception = entry->getException()) + auto time_in_microseconds = [](const time_point & timestamp) { - try - { - std::rethrow_exception(exception); - } - catch (const Exception & e) - { - arr_exception.push_back(e.displayText()); - } - catch (...) - { - arr_exception.push_back("Unknown exception"); - } + auto time_diff = duration_cast(steady_clock::now() - timestamp); + auto time_us = (system_clock::now() - time_diff).time_since_epoch().count(); + + DecimalUtils::DecimalComponents components{time_us / 1'000'000, time_us % 1'000'000}; + return DecimalField(DecimalUtils::decimalFromComponents(components, TIME_SCALE), TIME_SCALE); + }; + + const auto & insert_query = key.query->as(); + size_t i = 0; + + res_columns[i++]->insert(queryToString(insert_query)); + + /// If query is "INSERT INTO FUNCTION" then table_id is empty. + if (insert_query.table_id) + { + res_columns[i++]->insert(insert_query.table_id.getDatabaseName()); + res_columns[i++]->insert(insert_query.table_id.getTableName()); } else - arr_exception.push_back(""); - } + { + res_columns[i++]->insertDefault(); + res_columns[i++]->insertDefault(); + } - res_columns[i++]->insert(arr_query_id); - res_columns[i++]->insert(arr_bytes); - res_columns[i++]->insert(arr_finished); - res_columns[i++]->insert(arr_exception); + res_columns[i++]->insert(insert_query.format); + res_columns[i++]->insert(time_in_microseconds(first_update)); + res_columns[i++]->insert(data->size_in_bytes); + + Array arr_query_id; + Array arr_bytes; + + for (const auto & entry : data->entries) + { + arr_query_id.push_back(entry->query_id); + arr_bytes.push_back(entry->bytes.size()); + } + + res_columns[i++]->insert(arr_query_id); + res_columns[i++]->insert(arr_bytes); + } } } diff --git a/src/Storages/System/StorageSystemAsynchronousMetrics.cpp b/src/Storages/System/StorageSystemAsynchronousMetrics.cpp index e2f62b902b7..843c7cb85e1 100644 --- a/src/Storages/System/StorageSystemAsynchronousMetrics.cpp +++ b/src/Storages/System/StorageSystemAsynchronousMetrics.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index fa1c26b623d..0be44219c7d 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -195,21 +195,22 @@ void StorageSystemParts::processNextStorage( if (columns_mask[src_index++]) columns[res_index++]->insert(info.engine); - if (part->isStoredOnDisk()) + if (columns_mask[src_index++]) { - if (columns_mask[src_index++]) + if (part->isStoredOnDisk()) columns[res_index++]->insert(part->getDataPartStorage().getDiskName()); - if (columns_mask[src_index++]) - columns[res_index++]->insert(part->getDataPartStorage().getFullPath()); - } - else - { - if (columns_mask[src_index++]) - columns[res_index++]->insertDefault(); - if (columns_mask[src_index++]) + else columns[res_index++]->insertDefault(); } + if (columns_mask[src_index++]) + { + // The full path changes at clean up thread under deleting state, do not read it, avoid the race + if (part->isStoredOnDisk() && part_state != State::Deleting) + columns[res_index++]->insert(part->getDataPartStorage().getFullPath()); + else + columns[res_index++]->insertDefault(); + } { MinimalisticDataPartChecksums helper; diff --git a/src/Storages/System/StorageSystemPartsColumns.cpp b/src/Storages/System/StorageSystemPartsColumns.cpp index cd51c767eae..65b5af0c8e9 100644 --- a/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/src/Storages/System/StorageSystemPartsColumns.cpp @@ -192,7 +192,13 @@ void StorageSystemPartsColumns::processNextStorage( if (columns_mask[src_index++]) columns[res_index++]->insert(part->getDataPartStorage().getDiskName()); if (columns_mask[src_index++]) - columns[res_index++]->insert(part->getDataPartStorage().getFullPath()); + { + // The full path changes at clean up thread under deleting state, do not read it, avoid the race + if (part_state != State::Deleting) + columns[res_index++]->insert(part->getDataPartStorage().getFullPath()); + else + columns[res_index++]->insertDefault(); + } if (columns_mask[src_index++]) columns[res_index++]->insert(column.name); diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 72301a56d49..e1611f1ecfd 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -348,26 +348,26 @@ protected: res_columns[res_index++]->insert(static_cast(database->getObjectMetadataModificationTime(table_name))); { - Array dependencies_table_name_array; - Array dependencies_database_name_array; + Array views_table_name_array; + Array views_database_name_array; if (columns_mask[src_index] || columns_mask[src_index + 1]) { - const auto dependencies = DatabaseCatalog::instance().getDependencies(StorageID(database_name, table_name)); + const auto view_ids = DatabaseCatalog::instance().getDependentViews(StorageID(database_name, table_name)); - dependencies_table_name_array.reserve(dependencies.size()); - dependencies_database_name_array.reserve(dependencies.size()); - for (const auto & dependency : dependencies) + views_table_name_array.reserve(view_ids.size()); + views_database_name_array.reserve(view_ids.size()); + for (const auto & view_id : view_ids) { - dependencies_table_name_array.push_back(dependency.table_name); - dependencies_database_name_array.push_back(dependency.database_name); + views_table_name_array.push_back(view_id.table_name); + views_database_name_array.push_back(view_id.database_name); } } if (columns_mask[src_index++]) - res_columns[res_index++]->insert(dependencies_database_name_array); + res_columns[res_index++]->insert(views_database_name_array); if (columns_mask[src_index++]) - res_columns[res_index++]->insert(dependencies_table_name_array); + res_columns[res_index++]->insert(views_table_name_array); } if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2]) @@ -513,37 +513,38 @@ protected: if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2] || columns_mask[src_index + 3]) { - DependenciesInfo info = DatabaseCatalog::instance().getLoadingDependenciesInfo({database_name, table_name}); + auto dependencies = DatabaseCatalog::instance().getDependencies(StorageID{database_name, table_name}); + auto dependents = DatabaseCatalog::instance().getDependents(StorageID{database_name, table_name}); - Array loading_dependencies_databases; - Array loading_dependencies_tables; - loading_dependencies_databases.reserve(info.dependencies.size()); - loading_dependencies_tables.reserve(info.dependencies.size()); - for (auto && dependency : info.dependencies) + Array dependencies_databases; + Array dependencies_tables; + dependencies_databases.reserve(dependencies.size()); + dependencies_tables.reserve(dependencies.size()); + for (const auto & dependency : dependencies) { - loading_dependencies_databases.push_back(dependency.database); - loading_dependencies_tables.push_back(dependency.table); + dependencies_databases.push_back(dependency.database_name); + dependencies_tables.push_back(dependency.table_name); } - Array loading_dependent_databases; - Array loading_dependent_tables; - loading_dependent_databases.reserve(info.dependencies.size()); - loading_dependent_tables.reserve(info.dependencies.size()); - for (auto && dependent : info.dependent_database_objects) + Array dependents_databases; + Array dependents_tables; + dependents_databases.reserve(dependents.size()); + dependents_tables.reserve(dependents.size()); + for (const auto & dependent : dependents) { - loading_dependent_databases.push_back(dependent.database); - loading_dependent_tables.push_back(dependent.table); + dependents_databases.push_back(dependent.database_name); + dependents_tables.push_back(dependent.table_name); } if (columns_mask[src_index++]) - res_columns[res_index++]->insert(loading_dependencies_databases); + res_columns[res_index++]->insert(dependencies_databases); if (columns_mask[src_index++]) - res_columns[res_index++]->insert(loading_dependencies_tables); + res_columns[res_index++]->insert(dependencies_tables); if (columns_mask[src_index++]) - res_columns[res_index++]->insert(loading_dependent_databases); + res_columns[res_index++]->insert(dependents_databases); if (columns_mask[src_index++]) - res_columns[res_index++]->insert(loading_dependent_tables); + res_columns[res_index++]->insert(dependents_tables); } } diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index c0bc5ad8da9..442a7822e33 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -471,7 +471,7 @@ void StorageWindowView::alter( create_interpreter.setInternal(true); create_interpreter.execute(); - DatabaseCatalog::instance().addDependency(select_table_id, table_id); + DatabaseCatalog::instance().addViewDependency(select_table_id, table_id); shutdown_called = false; @@ -1566,7 +1566,7 @@ void StorageWindowView::writeIntoWindowView( void StorageWindowView::startup() { - DatabaseCatalog::instance().addDependency(select_table_id, getStorageID()); + DatabaseCatalog::instance().addViewDependency(select_table_id, getStorageID()); fire_task->activate(); clean_cache_task->activate(); @@ -1586,17 +1586,17 @@ void StorageWindowView::shutdown() fire_task->deactivate(); auto table_id = getStorageID(); - DatabaseCatalog::instance().removeDependency(select_table_id, table_id); + DatabaseCatalog::instance().removeViewDependency(select_table_id, table_id); } void StorageWindowView::checkTableCanBeDropped() const { auto table_id = getStorageID(); - Dependencies dependencies = DatabaseCatalog::instance().getDependencies(table_id); - if (!dependencies.empty()) + auto view_ids = DatabaseCatalog::instance().getDependentViews(table_id); + if (!view_ids.empty()) { - StorageID dependent_table_id = dependencies.front(); - throw Exception("Table has dependency " + dependent_table_id.getNameForLogs(), ErrorCodes::TABLE_WAS_NOT_DROPPED); + StorageID view_id = *view_ids.begin(); + throw Exception(ErrorCodes::TABLE_WAS_NOT_DROPPED, "Table has dependency {}", view_id); } } diff --git a/src/Storages/tests/gtest_named_collections.cpp b/src/Storages/tests/gtest_named_collections.cpp index 5ba9156bcd9..369e8ec44f6 100644 --- a/src/Storages/tests/gtest_named_collections.cpp +++ b/src/Storages/tests/gtest_named_collections.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -28,7 +29,7 @@ TEST(NamedCollections, SimpleConfig) Poco::AutoPtr document = dom_parser.parseString(xml); Poco::AutoPtr config = new Poco::Util::XMLConfiguration(document); - NamedCollectionFactory::instance().initialize(*config); + NamedCollectionUtils::loadFromConfig(*config); ASSERT_TRUE(NamedCollectionFactory::instance().exists("collection1")); ASSERT_TRUE(NamedCollectionFactory::instance().exists("collection2")); @@ -76,16 +77,16 @@ key5: 5 key6: 6.6 )CONFIG"); - collection2_copy->set("key4", "value44", true); - ASSERT_TRUE(collection2_copy->get("key4") == "value44"); - ASSERT_TRUE(collection2->get("key4") == "value4"); + collection2_copy->setOrUpdate("key4", "value44"); + ASSERT_EQ(collection2_copy->get("key4"), "value44"); + ASSERT_EQ(collection2->get("key4"), "value4"); collection2_copy->remove("key4"); - ASSERT_TRUE(collection2_copy->getOrDefault("key4", "N") == "N"); - ASSERT_TRUE(collection2->getOrDefault("key4", "N") == "value4"); + ASSERT_EQ(collection2_copy->getOrDefault("key4", "N"), "N"); + ASSERT_EQ(collection2->getOrDefault("key4", "N"), "value4"); - collection2_copy->set("key4", "value45"); - ASSERT_TRUE(collection2_copy->getOrDefault("key4", "N") == "value45"); + collection2_copy->setOrUpdate("key4", "value45"); + ASSERT_EQ(collection2_copy->getOrDefault("key4", "N"), "value45"); NamedCollectionFactory::instance().remove("collection2_copy"); ASSERT_FALSE(NamedCollectionFactory::instance().exists("collection2_copy")); @@ -97,7 +98,7 @@ TEST(NamedCollections, NestedConfig) { std::string xml(R"CONFIG( - + value1 @@ -110,21 +111,22 @@ TEST(NamedCollections, NestedConfig) - + )CONFIG"); Poco::XML::DOMParser dom_parser; Poco::AutoPtr document = dom_parser.parseString(xml); Poco::AutoPtr config = new Poco::Util::XMLConfiguration(document); - NamedCollectionFactory::instance().reload(*config); - ASSERT_TRUE(NamedCollectionFactory::instance().exists("collection1")); + NamedCollectionUtils::loadFromConfig(*config); - auto collection1 = NamedCollectionFactory::instance().get("collection1"); - ASSERT_TRUE(collection1 != nullptr); + ASSERT_TRUE(NamedCollectionFactory::instance().exists("collection3")); - ASSERT_EQ(collection1->dumpStructure(), + auto collection = NamedCollectionFactory::instance().get("collection3"); + ASSERT_TRUE(collection != nullptr); + + ASSERT_EQ(collection->dumpStructure(), R"CONFIG(key1: key1_1: value1 key2: @@ -135,9 +137,9 @@ key2: key2_5: 5 )CONFIG"); - ASSERT_EQ(collection1->get("key1.key1_1"), "value1"); - ASSERT_EQ(collection1->get("key2.key2_1"), "value2_1"); - ASSERT_EQ(collection1->get("key2.key2_2.key2_3.key2_4"), 4); - ASSERT_EQ(collection1->get("key2.key2_2.key2_3.key2_5"), 5); + ASSERT_EQ(collection->get("key1.key1_1"), "value1"); + ASSERT_EQ(collection->get("key2.key2_1"), "value2_1"); + ASSERT_EQ(collection->get("key2.key2_2.key2_3.key2_4"), 4); + ASSERT_EQ(collection->get("key2.key2_2.key2_3.key2_5"), 5); } diff --git a/tests/ci/.mypy.ini b/tests/ci/.mypy.ini new file mode 100644 index 00000000000..7326675067c --- /dev/null +++ b/tests/ci/.mypy.ini @@ -0,0 +1,16 @@ +[mypy] +warn_no_return = False +warn_unused_configs = True +disallow_subclassing_any = True +disallow_untyped_calls = False +disallow_untyped_defs = False +disallow_incomplete_defs = True +check_untyped_defs = True +disallow_untyped_decorators = True +no_implicit_optional = True +warn_redundant_casts = True +warn_unused_ignores = True +warn_return_any = True +no_implicit_reexport = True +strict_equality = True +strict_concatenate = True diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index d668dbe0498..c9e8dac2c00 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -121,7 +121,7 @@ def check_for_success_run( s3_prefix: str, build_name: str, build_config: BuildConfig, -): +) -> None: logged_prefix = os.path.join(S3_BUILDS_BUCKET, s3_prefix) logging.info("Checking for artifacts in %s", logged_prefix) try: @@ -174,7 +174,7 @@ def create_json_artifact( build_config: BuildConfig, elapsed: int, success: bool, -): +) -> None: subprocess.check_call( f"echo 'BUILD_URLS=build_urls_{build_name}' >> $GITHUB_ENV", shell=True ) @@ -218,7 +218,7 @@ def upload_master_static_binaries( build_config: BuildConfig, s3_helper: S3Helper, build_output_path: str, -): +) -> None: """Upload binary artifacts to a static S3 links""" static_binary_name = build_config.get("static_binary_name", False) if pr_info.number != 0: diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 58997bed253..1a2fdedefed 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -5,7 +5,7 @@ import logging import os import sys import time -from typing import List, Optional +from typing import Any, List, Optional import requests # type: ignore @@ -18,7 +18,7 @@ def get_with_retries( url: str, retries: int = DOWNLOAD_RETRIES_COUNT, sleep: int = 3, - **kwargs, + **kwargs: Any, ) -> requests.Response: logging.info( "Getting URL with %i tries and sleep %i in between: %s", retries, sleep, url @@ -41,18 +41,18 @@ def get_with_retries( return response -def get_build_name_for_check(check_name) -> str: - return CI_CONFIG["tests_config"][check_name]["required_build"] +def get_build_name_for_check(check_name: str) -> str: + return CI_CONFIG["tests_config"][check_name]["required_build"] # type: ignore -def read_build_urls(build_name, reports_path) -> List[str]: +def read_build_urls(build_name: str, reports_path: str) -> List[str]: for root, _, files in os.walk(reports_path): for f in files: if build_name in f: logging.info("Found build report json %s", f) with open(os.path.join(root, f), "r", encoding="utf-8") as file_handler: build_report = json.load(file_handler) - return build_report["build_urls"] + return build_report["build_urls"] # type: ignore return [] diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index 673b0204864..03e18d7766e 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -19,7 +19,7 @@ from env_helper import ( from report import create_build_html_report from s3_helper import S3Helper from get_robot_token import get_best_robot_token -from pr_info import PRInfo +from pr_info import NeedsDataType, PRInfo from commit_status_helper import ( get_commit, update_mergeable_check, @@ -28,7 +28,7 @@ from ci_config import CI_CONFIG from rerun_helper import RerunHelper -NEEDS_DATA_PATH = os.getenv("NEEDS_DATA_PATH") +NEEDS_DATA_PATH = os.getenv("NEEDS_DATA_PATH", "") class BuildResult: @@ -98,7 +98,7 @@ def get_failed_report( def process_report( - build_report, + build_report: dict, ) -> Tuple[List[BuildResult], List[List[str]], List[str]]: build_config = build_report["build_config"] build_result = BuildResult( @@ -144,16 +144,14 @@ def main(): os.makedirs(temp_path) build_check_name = sys.argv[1] - needs_data = None + needs_data = {} # type: NeedsDataType required_builds = 0 if os.path.exists(NEEDS_DATA_PATH): with open(NEEDS_DATA_PATH, "rb") as file_handler: needs_data = json.load(file_handler) required_builds = len(needs_data) - if needs_data is not None and all( - i["result"] == "skipped" for i in needs_data.values() - ): + if needs_data and all(i["result"] == "skipped" for i in needs_data.values()): logging.info("All builds are skipped, exiting") sys.exit(0) @@ -218,19 +216,21 @@ def main(): build_logs = [] for build_report in build_reports: - build_result, build_artifacts_url, build_logs_url = process_report(build_report) - logging.info( - "Got %s artifact groups for build report report", len(build_result) + _build_results, build_artifacts_url, build_logs_url = process_report( + build_report ) - build_results.extend(build_result) + logging.info( + "Got %s artifact groups for build report report", len(_build_results) + ) + build_results.extend(_build_results) build_artifacts.extend(build_artifacts_url) build_logs.extend(build_logs_url) for failed_job in missing_build_names: - build_result, build_artifacts_url, build_logs_url = get_failed_report( + _build_results, build_artifacts_url, build_logs_url = get_failed_report( failed_job ) - build_results.extend(build_result) + build_results.extend(_build_results) build_artifacts.extend(build_artifacts_url) build_logs.extend(build_logs_url) diff --git a/tests/queries/0_stateless/01676_long_clickhouse_client_autocomplete.reference b/tests/ci/cancel_and_rerun_workflow_lambda/__init__.py similarity index 100% rename from tests/queries/0_stateless/01676_long_clickhouse_client_autocomplete.reference rename to tests/ci/cancel_and_rerun_workflow_lambda/__init__.py diff --git a/tests/ci/cancel_and_rerun_workflow_lambda/app.py b/tests/ci/cancel_and_rerun_workflow_lambda/app.py index 6d63aaa141e..d93a9062a3b 100644 --- a/tests/ci/cancel_and_rerun_workflow_lambda/app.py +++ b/tests/ci/cancel_and_rerun_workflow_lambda/app.py @@ -13,8 +13,10 @@ import jwt import requests # type: ignore import boto3 # type: ignore +PULL_REQUEST_CI = "PullRequestCI" + NEED_RERUN_OR_CANCELL_WORKFLOWS = { - "PullRequestCI", + PULL_REQUEST_CI, "DocsCheck", "DocsReleaseChecks", "BackportPR", @@ -104,7 +106,7 @@ def _exec_get_with_retry(url: str, token: str) -> dict: try: response = requests.get(url, headers=headers) response.raise_for_status() - return response.json() + return response.json() # type: ignore except Exception as ex: print("Got exception executing request", ex) time.sleep(i + 1) @@ -114,13 +116,21 @@ def _exec_get_with_retry(url: str, token: str) -> dict: WorkflowDescription = namedtuple( "WorkflowDescription", - ["url", "run_id", "head_sha", "status", "rerun_url", "cancel_url", "conclusion"], + [ + "url", + "run_id", + "name", + "head_sha", + "status", + "rerun_url", + "cancel_url", + "conclusion", + ], ) def get_workflows_description_for_pull_request( - pull_request_event, - token, + pull_request_event: dict, token: str ) -> List[WorkflowDescription]: head_repo = pull_request_event["head"]["repo"]["full_name"] head_branch = pull_request_event["head"]["ref"] @@ -169,6 +179,7 @@ def get_workflows_description_for_pull_request( WorkflowDescription( url=workflow["url"], run_id=workflow["id"], + name=workflow["name"], head_sha=workflow["head_sha"], status=workflow["status"], rerun_url=workflow["rerun_url"], @@ -181,7 +192,7 @@ def get_workflows_description_for_pull_request( def get_workflow_description_fallback( - pull_request_event, token + pull_request_event: dict, token: str ) -> List[WorkflowDescription]: head_repo = pull_request_event["head"]["repo"]["full_name"] head_branch = pull_request_event["head"]["ref"] @@ -229,6 +240,7 @@ def get_workflow_description_fallback( WorkflowDescription( url=wf["url"], run_id=wf["id"], + name=wf["name"], head_sha=wf["head_sha"], status=wf["status"], rerun_url=wf["rerun_url"], @@ -241,11 +253,12 @@ def get_workflow_description_fallback( return workflow_descriptions -def get_workflow_description(workflow_url, token) -> WorkflowDescription: +def get_workflow_description(workflow_url: str, token: str) -> WorkflowDescription: workflow = _exec_get_with_retry(workflow_url, token) return WorkflowDescription( url=workflow["url"], run_id=workflow["id"], + name=workflow["name"], head_sha=workflow["head_sha"], status=workflow["status"], rerun_url=workflow["rerun_url"], @@ -268,8 +281,8 @@ def _exec_post_with_retry(url, token): raise Exception("Cannot execute POST request with retry") -def exec_workflow_url(urls_to_cancel, token): - for url in urls_to_cancel: +def exec_workflow_url(urls_to_post, token): + for url in urls_to_post: print("Post for workflow workflow using url", url) _exec_post_with_retry(url, token) print("Workflow post finished") @@ -289,7 +302,7 @@ def main(event): pull_request = event_data["pull_request"] labels = {label["name"] for label in pull_request["labels"]} print("PR has labels", labels) - if action == "closed" or "do not test" in labels: + if action == "closed" or (action == "labeled" and "do not test" in labels): print("PR merged/closed or manually labeled 'do not test' will kill workflows") workflow_descriptions = get_workflows_description_for_pull_request( pull_request, token @@ -307,6 +320,29 @@ def main(event): urls_to_cancel.append(workflow_description.cancel_url) print(f"Found {len(urls_to_cancel)} workflows to cancel") exec_workflow_url(urls_to_cancel, token) + return + elif action == "edited": + print("PR is edited, check if it needs to rerun") + workflow_descriptions = get_workflows_description_for_pull_request( + pull_request, token + ) + workflow_descriptions = ( + workflow_descriptions + or get_workflow_description_fallback(pull_request, token) + ) + workflow_descriptions.sort(key=lambda x: x.run_id) # type: ignore + most_recent_workflow = workflow_descriptions[-1] + if ( + most_recent_workflow.status == "completed" + and most_recent_workflow.name == PULL_REQUEST_CI + ): + print( + "The PR's body is changed and workflow is finished. " + "Rerun to check the description" + ) + exec_workflow_url([most_recent_workflow.rerun_url], token) + print("Rerun finished, exiting") + return elif action == "synchronize": print("PR is synchronized, going to stop old actions") workflow_descriptions = get_workflows_description_for_pull_request( @@ -339,8 +375,8 @@ def main(event): print("Not found any workflows") return - sorted_workflows = list(sorted(workflow_descriptions, key=lambda x: x.run_id)) - most_recent_workflow = sorted_workflows[-1] + workflow_descriptions.sort(key=lambda x: x.run_id) # type: ignore + most_recent_workflow = workflow_descriptions[-1] print("Latest workflow", most_recent_workflow) if ( most_recent_workflow.status != "completed" diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py index d1c9d3d394c..b3e90feef2a 100644 --- a/tests/ci/cherry_pick.py +++ b/tests/ci/cherry_pick.py @@ -79,7 +79,7 @@ Merge it only if you intend to backport changes to the target branch, otherwise self.backport_pr = None # type: Optional[PullRequest] self._backported = None # type: Optional[bool] self.git_prefix = ( # All commits to cherrypick are done as robot-clickhouse - "git -c user.email=robot-clickhouse@clickhouse.com " + "git -c user.email=robot-clickhouse@users.noreply.github.com " "-c user.name=robot-clickhouse -c commit.gpgsign=false" ) self.pre_check() @@ -92,7 +92,8 @@ Merge it only if you intend to backport changes to the target branch, otherwise if branch_updated: self._backported = True - def pop_prs(self, prs: PullRequests): + def pop_prs(self, prs: PullRequests) -> None: + """the method processes all prs and pops the ReleaseBranch related prs""" to_pop = [] # type: List[int] for i, pr in enumerate(prs): if self.name not in pr.head.ref: @@ -105,14 +106,14 @@ Merge it only if you intend to backport changes to the target branch, otherwise to_pop.append(i) else: logging.error( - "PR #%s doesn't head ref starting with known suffix", + "head ref of PR #%s isn't starting with known suffix", pr.number, ) for i in reversed(to_pop): # Going from the tail to keep the order and pop greater index first prs.pop(i) - def process(self, dry_run: bool): + def process(self, dry_run: bool) -> None: if self.backported: return if not self.cherrypick_pr: @@ -209,6 +210,7 @@ Merge it only if you intend to backport changes to the target branch, otherwise self._assign_new_pr(self.cherrypick_pr) def create_backport(self): + assert self.cherrypick_pr is not None # Checkout the backport branch from the remote and make all changes to # apply like they are only one cherry-pick commit on top of release git_runner(f"{self.git_prefix} checkout -f {self.backport_branch}") @@ -239,7 +241,7 @@ Merge it only if you intend to backport changes to the target branch, otherwise self.backport_pr.add_to_labels(Labels.BACKPORT) self._assign_new_pr(self.backport_pr) - def _assign_new_pr(self, new_pr: PullRequest): + def _assign_new_pr(self, new_pr: PullRequest) -> None: """Assign `new_pr` to author, merger and assignees of an original PR""" # It looks there some race when multiple .add_to_assignees are executed, # so we'll add all at once @@ -340,7 +342,7 @@ class Backport: ) self.error = e - def process_pr(self, pr: PullRequest): + def process_pr(self, pr: PullRequest) -> None: pr_labels = [label.name for label in pr.labels] if Labels.MUST_BACKPORT in pr_labels: branches = [ @@ -403,7 +405,7 @@ class Backport: # And check it after the running self.mark_pr_backported(pr) - def mark_pr_backported(self, pr: PullRequest): + def mark_pr_backported(self, pr: PullRequest) -> None: if self.dry_run: logging.info("DRY RUN: would mark PR #%s as done", pr.number) return @@ -488,7 +490,8 @@ def main(): gh = GitHub(token, per_page=100) bp = Backport(gh, args.repo, args.dry_run) - bp.gh.cache_path = str(f"{TEMP_PATH}/gh_cache") + # https://github.com/python/mypy/issues/3004 + bp.gh.cache_path = f"{TEMP_PATH}/gh_cache" # type: ignore bp.receive_release_prs() bp.receive_prs_for_backport() bp.process_backports() diff --git a/tests/ci/metrics_lambda/app.py b/tests/ci/ci_runners_metrics_lambda/app.py similarity index 68% rename from tests/ci/metrics_lambda/app.py rename to tests/ci/ci_runners_metrics_lambda/app.py index 4a1921bf312..2bc568bb462 100644 --- a/tests/ci/metrics_lambda/app.py +++ b/tests/ci/ci_runners_metrics_lambda/app.py @@ -1,18 +1,42 @@ #!/usr/bin/env python3 +""" +Lambda function to: + - calculate number of running runners + - cleaning dead runners from GitHub + - terminating stale lost runners in EC2 +""" import argparse import sys import json import time from collections import namedtuple +from datetime import datetime +from typing import Dict, List, Tuple import jwt -import requests -import boto3 -from botocore.exceptions import ClientError +import requests # type: ignore +import boto3 # type: ignore +from botocore.exceptions import ClientError # type: ignore + +UNIVERSAL_LABEL = "universal" +RUNNER_TYPE_LABELS = [ + "builder", + "func-tester", + "func-tester-aarch64", + "fuzzer-unit-tester", + "stress-tester", + "style-checker", + "style-checker-aarch64", +] + +RunnerDescription = namedtuple( + "RunnerDescription", ["id", "name", "tags", "offline", "busy"] +) +RunnerDescriptions = List[RunnerDescription] -def get_dead_runners_in_ec2(runners): +def get_dead_runners_in_ec2(runners: RunnerDescriptions) -> RunnerDescriptions: ids = { runner.name: runner for runner in runners @@ -74,9 +98,45 @@ def get_dead_runners_in_ec2(runners): return result_to_delete -def get_key_and_app_from_aws(): - import boto3 +def get_lost_ec2_instances(runners: RunnerDescriptions) -> List[dict]: + client = boto3.client("ec2") + reservations = client.describe_instances( + Filters=[{"Name": "tag-key", "Values": ["github:runner-type"]}] + )["Reservations"] + lost_instances = [] + # Here we refresh the runners to get the most recent state + now = datetime.now().timestamp() + for reservation in reservations: + for instance in reservation["Instances"]: + # Do not consider instances started 20 minutes ago as problematic + if now - instance["LaunchTime"].timestamp() < 1200: + continue + + runner_type = [ + tag["Value"] + for tag in instance["Tags"] + if tag["Key"] == "github:runner-type" + ][0] + # If there's no necessary labels in runner type it's fine + if not ( + UNIVERSAL_LABEL in runner_type or runner_type in RUNNER_TYPE_LABELS + ): + continue + + if instance["State"]["Name"] == "running" and ( + not [ + runner + for runner in runners + if runner.name == instance["InstanceId"] + ] + ): + lost_instances.append(instance) + + return lost_instances + + +def get_key_and_app_from_aws() -> Tuple[str, int]: secret_name = "clickhouse_github_secret_key" session = boto3.session.Session() client = session.client( @@ -92,7 +152,7 @@ def handler(event, context): main(private_key, app_id, True, True) -def get_installation_id(jwt_token): +def get_installation_id(jwt_token: str) -> int: headers = { "Authorization": f"Bearer {jwt_token}", "Accept": "application/vnd.github.v3+json", @@ -103,10 +163,12 @@ def get_installation_id(jwt_token): for installation in data: if installation["account"]["login"] == "ClickHouse": installation_id = installation["id"] - return installation_id + break + + return installation_id # type: ignore -def get_access_token(jwt_token, installation_id): +def get_access_token(jwt_token: str, installation_id: int) -> str: headers = { "Authorization": f"Bearer {jwt_token}", "Accept": "application/vnd.github.v3+json", @@ -117,34 +179,33 @@ def get_access_token(jwt_token, installation_id): ) response.raise_for_status() data = response.json() - return data["token"] + return data["token"] # type: ignore -RunnerDescription = namedtuple( - "RunnerDescription", ["id", "name", "tags", "offline", "busy"] -) - - -def list_runners(access_token): +def list_runners(access_token: str) -> RunnerDescriptions: headers = { "Authorization": f"token {access_token}", "Accept": "application/vnd.github.v3+json", } + per_page = 100 response = requests.get( - "https://api.github.com/orgs/ClickHouse/actions/runners?per_page=100", + f"https://api.github.com/orgs/ClickHouse/actions/runners?per_page={per_page}", headers=headers, ) response.raise_for_status() data = response.json() total_runners = data["total_count"] + print("Expected total runners", total_runners) runners = data["runners"] - total_pages = int(total_runners / 100 + 1) + # round to 0 for 0, 1 for 1..100, but to 2 for 101..200 + total_pages = (total_runners - 1) // per_page + 1 + print("Total pages", total_pages) for i in range(2, total_pages + 1): response = requests.get( "https://api.github.com/orgs/ClickHouse/actions/runners" - f"?page={i}&per_page=100", + f"?page={i}&per_page={per_page}", headers=headers, ) response.raise_for_status() @@ -167,33 +228,34 @@ def list_runners(access_token): return result -def group_runners_by_tag(listed_runners): - result = {} +def group_runners_by_tag( + listed_runners: RunnerDescriptions, +) -> Dict[str, RunnerDescriptions]: + result = {} # type: Dict[str, RunnerDescriptions] + + def add_to_result(tag, runner): + if tag not in result: + result[tag] = [] + result[tag].append(runner) - RUNNER_TYPE_LABELS = [ - "builder", - "func-tester", - "func-tester-aarch64", - "fuzzer-unit-tester", - "stress-tester", - "style-checker", - "style-checker-aarch64", - ] for runner in listed_runners: + if UNIVERSAL_LABEL in runner.tags: + # Do not proceed other labels if UNIVERSAL_LABEL is included + add_to_result(UNIVERSAL_LABEL, runner) + continue + for tag in runner.tags: if tag in RUNNER_TYPE_LABELS: - if tag not in result: - result[tag] = [] - result[tag].append(runner) + add_to_result(tag, runner) break else: - if "unlabeled" not in result: - result["unlabeled"] = [] - result["unlabeled"].append(runner) + add_to_result("unlabeled", runner) return result -def push_metrics_to_cloudwatch(listed_runners, namespace): +def push_metrics_to_cloudwatch( + listed_runners: RunnerDescriptions, namespace: str +) -> None: client = boto3.client("cloudwatch") metrics_data = [] busy_runners = sum( @@ -223,7 +285,7 @@ def push_metrics_to_cloudwatch(listed_runners, namespace): } ) if total_active_runners == 0: - busy_ratio = 100 + busy_ratio = 100.0 else: busy_ratio = busy_runners / total_active_runners * 100 @@ -238,7 +300,7 @@ def push_metrics_to_cloudwatch(listed_runners, namespace): client.put_metric_data(Namespace=namespace, MetricData=metrics_data) -def delete_runner(access_token, runner): +def delete_runner(access_token: str, runner: RunnerDescription) -> bool: headers = { "Authorization": f"token {access_token}", "Accept": "application/vnd.github.v3+json", @@ -250,10 +312,15 @@ def delete_runner(access_token, runner): ) response.raise_for_status() print(f"Response code deleting {runner.name} is {response.status_code}") - return response.status_code == 204 + return bool(response.status_code == 204) -def main(github_secret_key, github_app_id, push_to_cloudwatch, delete_offline_runners): +def main( + github_secret_key: str, + github_app_id: int, + push_to_cloudwatch: bool, + delete_offline_runners: bool, +) -> None: payload = { "iat": int(time.time()) - 60, "exp": int(time.time()) + (10 * 60), @@ -263,8 +330,8 @@ def main(github_secret_key, github_app_id, push_to_cloudwatch, delete_offline_ru encoded_jwt = jwt.encode(payload, github_secret_key, algorithm="RS256") installation_id = get_installation_id(encoded_jwt) access_token = get_access_token(encoded_jwt, installation_id) - runners = list_runners(access_token) - grouped_runners = group_runners_by_tag(runners) + gh_runners = list_runners(access_token) + grouped_runners = group_runners_by_tag(gh_runners) for group, group_runners in grouped_runners.items(): if push_to_cloudwatch: print(group) @@ -276,11 +343,18 @@ def main(github_secret_key, github_app_id, push_to_cloudwatch, delete_offline_ru if delete_offline_runners: print("Going to delete offline runners") - dead_runners = get_dead_runners_in_ec2(runners) + dead_runners = get_dead_runners_in_ec2(gh_runners) for runner in dead_runners: print("Deleting runner", runner) delete_runner(access_token, runner) + lost_instances = get_lost_ec2_instances(gh_runners) + if lost_instances: + print("Going to terminate lost runners") + ids = [i["InstanceId"] for i in lost_instances] + print("Terminating runners:", ids) + boto3.client("ec2").terminate_instances(InstanceIds=ids) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Get list of runners and their states") diff --git a/tests/ci/ci_runners_metrics_lambda/build_and_deploy_archive.sh b/tests/ci/ci_runners_metrics_lambda/build_and_deploy_archive.sh new file mode 120000 index 00000000000..96ba3fa024e --- /dev/null +++ b/tests/ci/ci_runners_metrics_lambda/build_and_deploy_archive.sh @@ -0,0 +1 @@ +../team_keys_lambda/build_and_deploy_archive.sh \ No newline at end of file diff --git a/tests/ci/metrics_lambda/requirements.txt b/tests/ci/ci_runners_metrics_lambda/requirements.txt similarity index 100% rename from tests/ci/metrics_lambda/requirements.txt rename to tests/ci/ci_runners_metrics_lambda/requirements.txt diff --git a/tests/ci/codebrowser_check.py b/tests/ci/codebrowser_check.py index 97036c6fc7b..412bcdf8818 100644 --- a/tests/ci/codebrowser_check.py +++ b/tests/ci/codebrowser_check.py @@ -7,14 +7,21 @@ import logging from github import Github -from env_helper import IMAGES_PATH, REPO_COPY, S3_TEST_REPORTS_BUCKET, S3_DOWNLOAD -from stopwatch import Stopwatch -from upload_result_helper import upload_results -from s3_helper import S3Helper -from get_robot_token import get_best_robot_token +from env_helper import ( + IMAGES_PATH, + REPO_COPY, + S3_DOWNLOAD, + S3_TEST_REPORTS_BUCKET, + TEMP_PATH, +) from commit_status_helper import post_commit_status from docker_pull_helper import get_image_with_version +from get_robot_token import get_best_robot_token +from pr_info import PRInfo +from s3_helper import S3Helper +from stopwatch import Stopwatch from tee_popen import TeePopen +from upload_result_helper import upload_results NAME = "Woboq Build" @@ -33,17 +40,16 @@ if __name__ == "__main__": stopwatch = Stopwatch() - temp_path = os.getenv("TEMP_PATH", os.path.abspath(".")) - gh = Github(get_best_robot_token(), per_page=100) + pr_info = PRInfo() - if not os.path.exists(temp_path): - os.makedirs(temp_path) + if not os.path.exists(TEMP_PATH): + os.makedirs(TEMP_PATH) docker_image = get_image_with_version(IMAGES_PATH, "clickhouse/codebrowser") s3_helper = S3Helper() - result_path = os.path.join(temp_path, "result_path") + result_path = os.path.join(TEMP_PATH, "result_path") if not os.path.exists(result_path): os.makedirs(result_path) @@ -51,7 +57,7 @@ if __name__ == "__main__": logging.info("Going to run codebrowser: %s", run_command) - run_log_path = os.path.join(temp_path, "runlog.log") + run_log_path = os.path.join(TEMP_PATH, "runlog.log") with TeePopen(run_command, run_log_path) as process: retcode = process.wait() @@ -60,7 +66,7 @@ if __name__ == "__main__": else: logging.info("Run failed") - subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) + subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {TEMP_PATH}", shell=True) report_path = os.path.join(result_path, "html_report") logging.info("Report path %s", report_path) @@ -76,12 +82,8 @@ if __name__ == "__main__": test_results = [(index_html, "Look at the report")] - report_url = upload_results( - s3_helper, 0, os.getenv("GITHUB_SHA"), test_results, [], NAME - ) + report_url = upload_results(s3_helper, 0, pr_info.sha, test_results, [], NAME) print(f"::notice ::Report url: {report_url}") - post_commit_status( - gh, os.getenv("GITHUB_SHA"), NAME, "Report built", "success", report_url - ) + post_commit_status(gh, pr_info.sha, NAME, "Report built", "success", report_url) diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 185dc64daa9..785250c3904 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -17,7 +17,7 @@ RETRY = 5 CommitStatuses = List[CommitStatus] -def override_status(status: str, check_name: str, invert=False) -> str: +def override_status(status: str, check_name: str, invert: bool = False) -> str: if CI_CONFIG["tests_config"].get(check_name, {}).get("force_tests", False): return "success" @@ -45,7 +45,7 @@ def get_commit(gh: Github, commit_sha: str, retry_count: int = RETRY) -> Commit: def post_commit_status( gh: Github, sha: str, check_name: str, description: str, state: str, report_url: str -): +) -> None: for i in range(RETRY): try: commit = get_commit(gh, sha, 1) @@ -64,7 +64,7 @@ def post_commit_status( def post_commit_status_to_file( file_path: str, description: str, state: str, report_url: str -): +) -> None: if os.path.exists(file_path): raise Exception(f'File "{file_path}" already exists!') with open(file_path, "w", encoding="utf-8") as f: @@ -88,21 +88,21 @@ def get_commit_filtered_statuses(commit: Commit) -> CommitStatuses: return list(filtered.values()) -def remove_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]): +def remove_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]) -> None: repo = gh.get_repo(GITHUB_REPOSITORY) pull_request = repo.get_pull(pr_info.number) for label in labels_names: pull_request.remove_from_labels(label) -def post_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]): +def post_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]) -> None: repo = gh.get_repo(GITHUB_REPOSITORY) pull_request = repo.get_pull(pr_info.number) for label in labels_names: pull_request.add_to_labels(label) -def fail_mergeable_check(commit: Commit, description: str): +def fail_mergeable_check(commit: Commit, description: str) -> None: commit.create_status( context="Mergeable Check", description=description, @@ -111,7 +111,7 @@ def fail_mergeable_check(commit: Commit, description: str): ) -def reset_mergeable_check(commit: Commit, description: str = ""): +def reset_mergeable_check(commit: Commit, description: str = "") -> None: commit.create_status( context="Mergeable Check", description=description, @@ -120,7 +120,7 @@ def reset_mergeable_check(commit: Commit, description: str = ""): ) -def update_mergeable_check(gh: Github, pr_info: PRInfo, check_name: str): +def update_mergeable_check(gh: Github, pr_info: PRInfo, check_name: str) -> None: if SKIP_MERGEABLE_CHECK_LABEL in pr_info.labels: return diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index fb7228628fd..0618969f94c 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -8,7 +8,7 @@ import shutil import subprocess import time import sys -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple, Union from github import Github @@ -52,7 +52,7 @@ class DockerImage: and self.only_amd64 == other.only_amd64 ) - def __lt__(self, other) -> bool: + def __lt__(self, other: Any) -> bool: if not isinstance(other, DockerImage): return False if self.parent and not other.parent: @@ -270,7 +270,7 @@ def build_and_push_one_image( def process_single_image( image: DockerImage, versions: List[str], - additional_cache, + additional_cache: str, push: bool, child: bool, ) -> List[Tuple[str, str, str]]: @@ -441,11 +441,15 @@ def main(): result_images = {} images_processing_result = [] + additional_cache = "" + if pr_info.release_pr or pr_info.merged_pr: + additional_cache = str(pr_info.release_pr or pr_info.merged_pr) + for image in changed_images: # If we are in backport PR, then pr_info.release_pr is defined # We use it as tag to reduce rebuilding time images_processing_result += process_image_with_parents( - image, image_versions, pr_info.release_pr, args.push + image, image_versions, additional_cache, args.push ) result_images[image.repo] = result_version diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index 09b7a99da78..2ba5a99de0a 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -70,7 +70,7 @@ def parse_args() -> argparse.Namespace: def load_images(path: str, suffix: str) -> Images: with open(os.path.join(path, CHANGED_IMAGES.format(suffix)), "rb") as images: - return json.load(images) + return json.load(images) # type: ignore def strip_suffix(suffix: str, images: Images) -> Images: diff --git a/tests/ci/docker_pull_helper.py b/tests/ci/docker_pull_helper.py index 04817ed7de3..5336966b3eb 100644 --- a/tests/ci/docker_pull_helper.py +++ b/tests/ci/docker_pull_helper.py @@ -6,11 +6,11 @@ import time import subprocess import logging -from typing import Optional +from typing import List, Optional class DockerImage: - def __init__(self, name, version: Optional[str] = None): + def __init__(self, name: str, version: Optional[str] = None): self.name = name if version is None: self.version = "latest" @@ -22,8 +22,11 @@ class DockerImage: def get_images_with_versions( - reports_path, required_image, pull=True, version: Optional[str] = None -): + reports_path: str, + required_images: List[str], + pull: bool = True, + version: Optional[str] = None, +) -> List[DockerImage]: images_path = None for root, _, files in os.walk(reports_path): for f in files: @@ -45,12 +48,13 @@ def get_images_with_versions( images = {} docker_images = [] - for image_name in required_image: + for image_name in required_images: docker_image = DockerImage(image_name, version) if image_name in images: docker_image.version = images[image_name] docker_images.append(docker_image) + latest_error = Exception("predefined to avoid access before created") if pull: for docker_image in docker_images: for i in range(10): @@ -75,6 +79,8 @@ def get_images_with_versions( return docker_images -def get_image_with_version(reports_path, image, pull=True, version=None): +def get_image_with_version( + reports_path: str, image: str, pull: bool = True, version: Optional[str] = None +) -> DockerImage: logging.info("Looking for images file in %s", reports_path) return get_images_with_versions(reports_path, [image], pull, version=version)[0] diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index 1848300e2f6..8b18a580ed7 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -43,55 +43,55 @@ class TestDockerImageCheck(unittest.TestCase): "docker/test/stateless", "clickhouse/stateless-test", False, - "clickhouse/test-base", + "clickhouse/test-base", # type: ignore ), di.DockerImage( "docker/test/integration/base", "clickhouse/integration-test", False, - "clickhouse/test-base", + "clickhouse/test-base", # type: ignore ), di.DockerImage( "docker/test/fuzzer", "clickhouse/fuzzer", False, - "clickhouse/test-base", + "clickhouse/test-base", # type: ignore ), di.DockerImage( "docker/test/keeper-jepsen", "clickhouse/keeper-jepsen-test", False, - "clickhouse/test-base", + "clickhouse/test-base", # type: ignore ), di.DockerImage( "docker/docs/check", "clickhouse/docs-check", False, - "clickhouse/docs-builder", + "clickhouse/docs-builder", # type: ignore ), di.DockerImage( "docker/docs/release", "clickhouse/docs-release", False, - "clickhouse/docs-builder", + "clickhouse/docs-builder", # type: ignore ), di.DockerImage( "docker/test/stateful", "clickhouse/stateful-test", False, - "clickhouse/stateless-test", + "clickhouse/stateless-test", # type: ignore ), di.DockerImage( "docker/test/unit", "clickhouse/unit-test", False, - "clickhouse/stateless-test", + "clickhouse/stateless-test", # type: ignore ), di.DockerImage( "docker/test/stress", "clickhouse/stress-test", False, - "clickhouse/stateful-test", + "clickhouse/stateful-test", # type: ignore ), ] ) @@ -277,7 +277,7 @@ class TestDockerServer(unittest.TestCase): ds.gen_tags(version, "auto") @patch("docker_server.get_tagged_versions") - def test_auto_release_type(self, mock_tagged_versions: MagicMock): + def test_auto_release_type(self, mock_tagged_versions: MagicMock) -> None: mock_tagged_versions.return_value = [ get_version_from_string("1.1.1.1"), get_version_from_string("1.2.1.1"), diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index a18f47497fd..ab0c3c6f688 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -42,11 +42,13 @@ def GITHUB_JOB_ID() -> str: if _GITHUB_JOB_ID: return _GITHUB_JOB_ID jobs = [] + page = 1 while not _GITHUB_JOB_ID: response = get_with_retries( f"https://api.github.com/repos/{GITHUB_REPOSITORY}/" - f"actions/runs/{GITHUB_RUN_ID}/jobs?per_page=100" + f"actions/runs/{GITHUB_RUN_ID}/jobs?per_page=100&page={page}" ) + page += 1 data = response.json() jobs.extend(data["jobs"]) for job in data["jobs"]: @@ -55,7 +57,10 @@ def GITHUB_JOB_ID() -> str: _GITHUB_JOB_ID = job["id"] _GITHUB_JOB_URL = job["html_url"] return _GITHUB_JOB_ID - if len(jobs) == data["total_count"]: + if ( + len(jobs) >= data["total_count"] # just in case of inconsistency + or len(data["jobs"]) == 0 # if we excided pages + ): _GITHUB_JOB_ID = "0" return _GITHUB_JOB_ID diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 03e42726808..2a6a0d5fa57 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -6,6 +6,7 @@ import os import csv import sys import atexit +from typing import List, Tuple from github import Github @@ -50,8 +51,10 @@ def get_fasttest_cmd( ) -def process_results(result_folder): - test_results = [] +def process_results( + result_folder: str, +) -> Tuple[str, str, List[Tuple[str, str]], List[str]]: + test_results = [] # type: List[Tuple[str, str]] additional_files = [] # Just upload all files from result_folder. # If task provides processed results, then it's responsible for content of @@ -78,7 +81,7 @@ def process_results(result_folder): results_path = os.path.join(result_folder, "test_results.tsv") if os.path.exists(results_path): with open(results_path, "r", encoding="utf-8") as results_file: - test_results = list(csv.reader(results_file, delimiter="\t")) + test_results = list(csv.reader(results_file, delimiter="\t")) # type: ignore if len(test_results) == 0: return "error", "Empty test_results.tsv", test_results, additional_files @@ -172,7 +175,7 @@ if __name__ == "__main__": "test_log.txt" in test_output_files or "test_result.txt" in test_output_files ) test_result_exists = "test_results.tsv" in test_output_files - test_results = [] + test_results = [] # type: List[Tuple[str, str]] if "submodule_log.txt" not in test_output_files: description = "Cannot clone repository" state = "failure" diff --git a/tests/ci/finish_check.py b/tests/ci/finish_check.py index a0b7f14ecfb..ea2f5eb3136 100644 --- a/tests/ci/finish_check.py +++ b/tests/ci/finish_check.py @@ -5,27 +5,11 @@ from github import Github from env_helper import GITHUB_RUN_URL from pr_info import PRInfo from get_robot_token import get_best_robot_token -from commit_status_helper import get_commit +from commit_status_helper import get_commit, get_commit_filtered_statuses NAME = "Run Check" -def filter_statuses(statuses): - """ - Squash statuses to latest state - 1. context="first", state="success", update_time=1 - 2. context="second", state="success", update_time=2 - 3. context="first", stat="failure", update_time=3 - =========> - 1. context="second", state="success" - 2. context="first", stat="failure" - """ - filt = {} - for status in sorted(statuses, key=lambda x: x.updated_at): - filt[status.context] = status - return filt - - if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -34,8 +18,13 @@ if __name__ == "__main__": commit = get_commit(gh, pr_info.sha) url = GITHUB_RUN_URL - statuses = filter_statuses(list(commit.get_statuses())) - if NAME in statuses and statuses[NAME].state == "pending": + statuses = get_commit_filtered_statuses(commit) + pending_status = any( # find NAME status in pending state + True + for status in statuses + if status.context == NAME and status.state == "pending" + ) + if pending_status: commit.create_status( context=NAME, description="All checks finished", diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index f7d3288c316..87833d688af 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -7,6 +7,7 @@ import os import subprocess import sys import atexit +from typing import List, Tuple from github import Github @@ -122,8 +123,11 @@ def get_tests_to_run(pr_info): return list(result) -def process_results(result_folder, server_log_path): - test_results = [] +def process_results( + result_folder: str, + server_log_path: str, +) -> Tuple[str, str, List[Tuple[str, str]], List[str]]: + test_results = [] # type: List[Tuple[str, str]] additional_files = [] # Just upload all files from result_folder. # If task provides processed results, then it's responsible for content of result_folder. @@ -166,7 +170,7 @@ def process_results(result_folder, server_log_path): return "error", "Not found test_results.tsv", test_results, additional_files with open(results_path, "r", encoding="utf-8") as results_file: - test_results = list(csv.reader(results_file, delimiter="\t")) + test_results = list(csv.reader(results_file, delimiter="\t")) # type: ignore if len(test_results) == 0: return "error", "Empty test_results.tsv", test_results, additional_files @@ -232,8 +236,8 @@ if __name__ == "__main__": sys.exit(0) if "RUN_BY_HASH_NUM" in os.environ: - run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM")) - run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL")) + run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM", "0")) + run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL", "0")) check_name_with_group = ( check_name + f" [{run_by_hash_num + 1}/{run_by_hash_total}]" ) diff --git a/tests/ci/get_previous_release_tag.py b/tests/ci/get_previous_release_tag.py index bfce69a17d9..b9ad51379d2 100755 --- a/tests/ci/get_previous_release_tag.py +++ b/tests/ci/get_previous_release_tag.py @@ -3,7 +3,7 @@ import re import logging -import requests +import requests # type: ignore CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" VERSION_PATTERN = r"(v(?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)" diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 163e1ce071e..6ecaf468ed1 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -1,8 +1,17 @@ #!/usr/bin/env python3 import logging +from dataclasses import dataclass import boto3 # type: ignore -from github import Github # type: ignore +from github import Github +from github.AuthenticatedUser import AuthenticatedUser + + +@dataclass +class Token: + user: AuthenticatedUser + value: str + rest: int def get_parameter_from_ssm(name, decrypt=True, client=None): @@ -19,7 +28,7 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): ] )["Parameters"] assert parameters - token = {"login": "", "value": "", "rest": 0} + token = None for token_name in [p["Name"] for p in parameters]: value = get_parameter_from_ssm(token_name, True, client) @@ -29,12 +38,15 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): user = gh.get_user() rest, _ = gh.rate_limiting logging.info("Get token with %s remaining requests", rest) - if token["rest"] < rest: - token = {"user": user, "value": value, "rest": rest} + if token is None: + token = Token(user, value, rest) + continue + if token.rest < rest: + token.user, token.value, token.rest = user, value, rest - assert token["value"] + assert token logging.info( - "User %s with %s remaining requests is used", token["user"].login, token["rest"] + "User %s with %s remaining requests is used", token.user.login, token.rest ) - return token["value"] + return token.value diff --git a/tests/ci/git_helper.py b/tests/ci/git_helper.py index 77c2fc9cf05..eb5e835eab3 100644 --- a/tests/ci/git_helper.py +++ b/tests/ci/git_helper.py @@ -4,7 +4,7 @@ import logging import os.path as p import re import subprocess -from typing import List, Optional +from typing import Any, List, Optional logger = logging.getLogger(__name__) @@ -21,19 +21,19 @@ TWEAK = 1 # Py 3.8 removeprefix and removesuffix -def removeprefix(string: str, prefix: str): +def removeprefix(string: str, prefix: str) -> str: if string.startswith(prefix): return string[len(prefix) :] # noqa: ignore E203, false positive return string -def removesuffix(string: str, suffix: str): +def removesuffix(string: str, suffix: str) -> str: if string.endswith(suffix): return string[: -len(suffix)] return string -def commit(name: str): +def commit(name: str) -> str: r = re.compile(SHA_REGEXP) if not r.match(name): raise argparse.ArgumentTypeError( @@ -42,7 +42,7 @@ def commit(name: str): return name -def release_branch(name: str): +def release_branch(name: str) -> str: r = re.compile(RELEASE_BRANCH_REGEXP) if not r.match(name): raise argparse.ArgumentTypeError("release branch should be as 12.1") @@ -55,20 +55,23 @@ class Runner: def __init__(self, cwd: str = CWD): self._cwd = cwd - def run(self, cmd: str, cwd: Optional[str] = None, **kwargs) -> str: + def run(self, cmd: str, cwd: Optional[str] = None, **kwargs: Any) -> str: if cwd is None: cwd = self.cwd logger.debug("Running command: %s", cmd) - return subprocess.check_output( - cmd, shell=True, cwd=cwd, encoding="utf-8", **kwargs - ).strip() + output = str( + subprocess.check_output( + cmd, shell=True, cwd=cwd, encoding="utf-8", **kwargs + ).strip() + ) + return output @property def cwd(self) -> str: return self._cwd @cwd.setter - def cwd(self, value: str): + def cwd(self, value: str) -> None: # Set _cwd only once, then set it to readonly if self._cwd != CWD: return @@ -139,7 +142,7 @@ class Git: ) @staticmethod - def check_tag(value: str): + def check_tag(value: str) -> None: if value == "": return if not Git._tag_pattern.match(value): @@ -150,7 +153,7 @@ class Git: return self._latest_tag @latest_tag.setter - def latest_tag(self, value: str): + def latest_tag(self, value: str) -> None: self.check_tag(value) self._latest_tag = value @@ -159,7 +162,7 @@ class Git: return self._new_tag @new_tag.setter - def new_tag(self, value: str): + def new_tag(self, value: str) -> None: self.check_tag(value) self._new_tag = value diff --git a/tests/ci/github_helper.py b/tests/ci/github_helper.py index 685d9f2c841..bd740827b34 100644 --- a/tests/ci/github_helper.py +++ b/tests/ci/github_helper.py @@ -8,11 +8,18 @@ from time import sleep from typing import List, Optional, Tuple import github -from github.GithubException import RateLimitExceededException -from github.Issue import Issue -from github.NamedUser import NamedUser -from github.PullRequest import PullRequest -from github.Repository import Repository + +# explicit reimport +# pylint: disable=useless-import-alias +from github.GithubException import ( + RateLimitExceededException as RateLimitExceededException, +) +from github.Issue import Issue as Issue +from github.NamedUser import NamedUser as NamedUser +from github.PullRequest import PullRequest as PullRequest +from github.Repository import Repository as Repository + +# pylint: enable=useless-import-alias CACHE_PATH = p.join(p.dirname(p.realpath(__file__)), "gh_cache") @@ -90,7 +97,7 @@ class GitHub(github.Github): raise exception # pylint: enable=signature-differs - def get_pulls_from_search(self, *args, **kwargs) -> PullRequests: + def get_pulls_from_search(self, *args, **kwargs) -> PullRequests: # type: ignore """The search api returns actually issues, so we need to fetch PullRequests""" issues = self.search_issues(*args, **kwargs) repos = {} @@ -168,7 +175,7 @@ class GitHub(github.Github): self.dump(user, prfd) # type: ignore return user - def _get_cached(self, path: Path): + def _get_cached(self, path: Path): # type: ignore with open(path, "rb") as ob_fd: return self.load(ob_fd) # type: ignore @@ -190,11 +197,11 @@ class GitHub(github.Github): return False, cached_obj @property - def cache_path(self): + def cache_path(self) -> Path: return self._cache_path @cache_path.setter - def cache_path(self, value: str): + def cache_path(self, value: str) -> None: self._cache_path = Path(value) if self._cache_path.exists(): assert self._cache_path.is_dir() @@ -208,5 +215,6 @@ class GitHub(github.Github): return self._retries @retries.setter - def retries(self, value: int): + def retries(self, value: int) -> None: + assert isinstance(value, int) self._retries = value diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index cba428cbcf5..e61117a4b45 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -7,6 +7,7 @@ import logging import os import subprocess import sys +from typing import List, Tuple from github import Github @@ -87,8 +88,10 @@ def get_env_for_runner(build_path, repo_path, result_path, work_path): return my_env -def process_results(result_folder): - test_results = [] +def process_results( + result_folder: str, +) -> Tuple[str, str, List[Tuple[str, str]], List[str]]: + test_results = [] # type: List[Tuple[str, str]] additional_files = [] # Just upload all files from result_folder. # If task provides processed results, then it's responsible for content of result_folder. @@ -115,7 +118,7 @@ def process_results(result_folder): results_path = os.path.join(result_folder, "test_results.tsv") if os.path.exists(results_path): with open(results_path, "r", encoding="utf-8") as results_file: - test_results = list(csv.reader(results_file, delimiter="\t")) + test_results = list(csv.reader(results_file, delimiter="\t")) # type: ignore if len(test_results) == 0: return "error", "Empty test_results.tsv", test_results, additional_files @@ -153,8 +156,8 @@ if __name__ == "__main__": validate_bugix_check = args.validate_bugfix if "RUN_BY_HASH_NUM" in os.environ: - run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM")) - run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL")) + run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM", "0")) + run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL", "0")) check_name_with_group = ( check_name + f" [{run_by_hash_num + 1}/{run_by_hash_total}]" ) diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py index 4116d15bba6..69964c0a0bc 100644 --- a/tests/ci/jepsen_check.py +++ b/tests/ci/jepsen_check.py @@ -7,9 +7,9 @@ import sys import argparse -import boto3 +import boto3 # type: ignore +import requests # type: ignore from github import Github -import requests from env_helper import REPO_COPY, TEMP_PATH, S3_BUILDS_BUCKET, S3_DOWNLOAD from stopwatch import Stopwatch diff --git a/tests/ci/mark_release_ready.py b/tests/ci/mark_release_ready.py index be1771e62bd..57ddb166693 100644 --- a/tests/ci/mark_release_ready.py +++ b/tests/ci/mark_release_ready.py @@ -5,8 +5,7 @@ from env_helper import GITHUB_JOB_URL from get_robot_token import get_best_robot_token from github_helper import GitHub from pr_info import PRInfo - -RELEASE_READY_STATUS = "Ready for release" +from release import RELEASE_READY_STATUS def main(): diff --git a/tests/ci/metrics_lambda/Dockerfile b/tests/ci/metrics_lambda/Dockerfile deleted file mode 100644 index 0d50224c51d..00000000000 --- a/tests/ci/metrics_lambda/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM public.ecr.aws/lambda/python:3.9 - -# Install the function's dependencies using file requirements.txt -# from your project folder. - -COPY requirements.txt . -RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}" - -# Copy function code -COPY app.py ${LAMBDA_TASK_ROOT} - -# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) -CMD [ "app.handler" ] diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index 78cf9fad001..acde5be5814 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -112,6 +112,16 @@ if __name__ == "__main__": else: check_name_with_group = check_name + is_aarch64 = "aarch64" in os.getenv("CHECK_NAME", "Performance Comparison").lower() + if pr_info.number != 0 and is_aarch64 and "pr-performance" not in pr_info.labels: + status = "success" + message = "Skipped, not labeled with 'pr-performance'" + report_url = GITHUB_RUN_URL + post_commit_status( + gh, pr_info.sha, check_name_with_group, message, status, report_url + ) + sys.exit(0) + test_grep_exclude_filter = CI_CONFIG["tests_config"][check_name][ "test_grep_exclude_filter" ] diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 5f725a61b3e..ddeb070b2b9 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -2,7 +2,7 @@ import json import logging import os -from typing import Set +from typing import Dict, List, Set, Union from unidiff import PatchSet # type: ignore @@ -16,6 +16,7 @@ from env_helper import ( FORCE_TESTS_LABEL = "force tests" SKIP_MERGEABLE_CHECK_LABEL = "skip mergeable check" +NeedsDataType = Dict[str, Dict[str, Union[str, Dict[str, str]]]] DIFF_IN_DOCUMENTATION_EXT = [ ".html", @@ -46,15 +47,22 @@ def get_pr_for_commit(sha, ref): try: response = get_with_retries(try_get_pr_url, sleep=RETRY_SLEEP) data = response.json() + our_prs = [] # type: List[Dict] if len(data) > 1: print("Got more than one pr for commit", sha) for pr in data: + # We need to check if the PR is created in our repo, because + # https://github.com/kaynewu/ClickHouse/pull/2 + # has broke our PR search once in a while + if pr["base"]["repo"]["full_name"] != GITHUB_REPOSITORY: + continue # refs for pushes looks like refs/head/XX # refs for RPs looks like XX if pr["head"]["ref"] in ref: return pr + our_prs.append(pr) print("Cannot find PR with required ref", ref, "returning first one") - first_pr = data[0] + first_pr = our_prs[0] return first_pr except Exception as ex: print("Cannot fetch PR info from commit", ex) @@ -64,6 +72,7 @@ def get_pr_for_commit(sha, ref): class PRInfo: default_event = { "commits": 1, + "head_commit": {"message": "commit_message"}, "before": "HEAD~", "after": "HEAD", "ref": None, @@ -86,7 +95,9 @@ class PRInfo: self.changed_files = set() # type: Set[str] self.body = "" self.diff_urls = [] + # release_pr and merged_pr are used for docker images additional cache self.release_pr = 0 + self.merged_pr = 0 ref = github_event.get("ref", "refs/heads/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] @@ -143,7 +154,7 @@ class PRInfo: self.body = github_event["pull_request"]["body"] self.labels = { label["name"] for label in github_event["pull_request"]["labels"] - } + } # type: Set[str] self.user_login = github_event["pull_request"]["user"]["login"] self.user_orgs = set([]) @@ -158,6 +169,14 @@ class PRInfo: self.diff_urls.append(github_event["pull_request"]["diff_url"]) elif "commits" in github_event: + # `head_commit` always comes with `commits` + commit_message = github_event["head_commit"]["message"] + if commit_message.startswith("Merge pull request #"): + merged_pr = commit_message.split(maxsplit=4)[3] + try: + self.merged_pr = int(merged_pr[1:]) + except ValueError: + logging.error("Failed to convert %s to integer", merged_pr) self.sha = github_event["after"] pull_request = get_pr_for_commit(self.sha, github_event["ref"]) repo_prefix = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}" @@ -167,7 +186,7 @@ class PRInfo: if pull_request is None or pull_request["state"] == "closed": # it's merged PR to master self.number = 0 - self.labels = {} + self.labels = set() self.pr_html_url = f"{repo_prefix}/commits/{ref}" self.base_ref = ref self.base_name = self.repo_full_name @@ -217,7 +236,7 @@ class PRInfo: print(json.dumps(github_event, sort_keys=True, indent=4)) self.sha = os.getenv("GITHUB_SHA") self.number = 0 - self.labels = {} + self.labels = set() repo_prefix = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}" self.task_url = GITHUB_RUN_URL self.commit_html_url = f"{repo_prefix}/commits/{self.sha}" diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index dd8081227bf..97971f207ce 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -5,7 +5,7 @@ import logging import os import re from collections import namedtuple -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple from artifactory import ArtifactorySaaSPath # type: ignore from build_download_helper import download_build_with_progress @@ -14,7 +14,7 @@ from git_helper import TAG_REGEXP, commit, removeprefix, removesuffix # Necessary ENV variables -def getenv(name: str, default: str = None): +def getenv(name: str, default: Optional[str] = None) -> str: env = os.getenv(name, default) if env is not None: return env @@ -62,7 +62,7 @@ class Packages: raise ValueError(f"{deb_pkg} not in {self.deb}") return removesuffix(deb_pkg, ".deb").split("_")[-1] - def replace_with_fallback(self, name: str): + def replace_with_fallback(self, name: str) -> None: if name.endswith(".deb"): suffix = self.deb.pop(name) self.deb[self.fallback_to_all(name)] = self.fallback_to_all(suffix) @@ -80,7 +80,7 @@ class Packages: return os.path.join(TEMP_PATH, package_file) @staticmethod - def fallback_to_all(url_or_name: str): + def fallback_to_all(url_or_name: str) -> str: """Until July 2022 we had clickhouse-server and clickhouse-client with arch 'all'""" # deb @@ -111,7 +111,7 @@ class S3: self.force_download = force_download self.packages = Packages(version) - def download_package(self, package_file: str, s3_path_suffix: str): + def download_package(self, package_file: str, s3_path_suffix: str) -> None: path = Packages.path(package_file) fallback_path = Packages.fallback_to_all(path) if not self.force_download and ( @@ -186,7 +186,12 @@ class Release: class Artifactory: def __init__( - self, url: str, release: str, deb_repo="deb", rpm_repo="rpm", tgz_repo="tgz" + self, + url: str, + release: str, + deb_repo: str = "deb", + rpm_repo: str = "rpm", + tgz_repo: str = "tgz", ): self._url = url self._release = release @@ -196,7 +201,7 @@ class Artifactory: # check the credentials ENVs for early exit self.__path_helper("_deb", "") - def deploy_deb(self, packages: Packages): + def deploy_deb(self, packages: Packages) -> None: for package_file in packages.deb: path = packages.path(package_file) dist = self._release @@ -212,13 +217,13 @@ class Artifactory: ) self.deb_path(package_file).deploy_deb(path, dist, comp, arch) - def deploy_rpm(self, packages: Packages): + def deploy_rpm(self, packages: Packages) -> None: for package_file in packages.rpm: path = packages.path(package_file) logging.info("Deploy %s to artifactory", path) self.rpm_path(package_file).deploy_file(path) - def deploy_tgz(self, packages: Packages): + def deploy_tgz(self, packages: Packages) -> None: for package_file in packages.tgz: path = packages.path(package_file) logging.info("Deploy %s to artifactory", path) @@ -316,19 +321,19 @@ def parse_args() -> argparse.Namespace: return args -def process_deb(s3: S3, art_clients: List[Artifactory]): +def process_deb(s3: S3, art_clients: List[Artifactory]) -> None: s3.download_deb() for art_client in art_clients: art_client.deploy_deb(s3.packages) -def process_rpm(s3: S3, art_clients: List[Artifactory]): +def process_rpm(s3: S3, art_clients: List[Artifactory]) -> None: s3.download_rpm() for art_client in art_clients: art_client.deploy_rpm(s3.packages) -def process_tgz(s3: S3, art_clients: List[Artifactory]): +def process_tgz(s3: S3, art_clients: List[Artifactory]) -> None: s3.download_tgz() for art_client in art_clients: art_client.deploy_tgz(s3.packages) diff --git a/tests/ci/release.py b/tests/ci/release.py index 8024091e300..8e58413f91f 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ script to create releases for ClickHouse @@ -11,14 +11,13 @@ On another hand, PyGithub is used for convenient getting commit's status from AP from contextlib import contextmanager -from typing import List, Optional +from typing import Any, Iterator, List, Literal, Optional import argparse +import json import logging import subprocess from git_helper import commit, release_branch -from github_helper import GitHub -from mark_release_ready import RELEASE_READY_STATUS from version_helper import ( FILE_WITH_VERSION_PATH, GENERATED_CONTRIBUTORS, @@ -31,6 +30,7 @@ from version_helper import ( update_contributors, ) +RELEASE_READY_STATUS = "Ready for release" git = Git() @@ -48,7 +48,7 @@ class Repo: return self._url @url.setter - def url(self, protocol: str): + def url(self, protocol: str) -> None: if protocol == "ssh": self._url = f"git@github.com:{self}.git" elif protocol == "https": @@ -68,17 +68,23 @@ class Release: CMAKE_PATH = get_abs_path(FILE_WITH_VERSION_PATH) CONTRIBUTORS_PATH = get_abs_path(GENERATED_CONTRIBUTORS) - def __init__(self, repo: Repo, release_commit: str, release_type: str): + def __init__( + self, + repo: Repo, + release_commit: str, + release_type: Literal["major", "minor", "patch"], + ): self.repo = repo self._release_commit = "" self.release_commit = release_commit + assert release_type in self.BIG + self.SMALL self.release_type = release_type self._git = git self._version = get_version_from_repo(git=self._git) self._release_branch = "" self._rollback_stack = [] # type: List[str] - def run(self, cmd: str, cwd: Optional[str] = None, **kwargs) -> str: + def run(self, cmd: str, cwd: Optional[str] = None, **kwargs: Any) -> str: cwd_text = "" if cwd: cwd_text = f" (CWD='{cwd}')" @@ -106,31 +112,30 @@ class Release: return VersionType.STABLE def check_commit_release_ready(self): - # First, get the auth token from gh cli - auth_status = self.run( - "gh auth status -t", stderr=subprocess.STDOUT - ).splitlines() - token = "" - for line in auth_status: - if "✓ Token:" in line: - token = line.split()[-1] - if not token: - logging.error("Can not extract token from `gh auth`") - raise subprocess.SubprocessError("Can not extract token from `gh auth`") - gh = GitHub(token, per_page=100) - repo = gh.get_repo(str(self.repo)) + per_page = 100 + page = 1 + while True: + statuses = json.loads( + self.run( + f"gh api 'repos/{self.repo}/commits/{self.release_commit}" + f"/statuses?per_page={per_page}&page={page}'" + ) + ) + + if not statuses: + break + + for status in statuses: + if status["context"] == RELEASE_READY_STATUS: + if not status["state"] == "success": + raise Exception( + f"the status {RELEASE_READY_STATUS} is {status['state']}" + ", not success" + ) - # Statuses are ordered by descending updated_at, so the first necessary - # status in the list is the most recent - statuses = repo.get_commit(self.release_commit).get_statuses() - for status in statuses: - if status.context == RELEASE_READY_STATUS: - if status.state == "success": return - raise Exception( - f"the status {RELEASE_READY_STATUS} is {status.state}, not success" - ) + page += 1 raise Exception( f"the status {RELEASE_READY_STATUS} " @@ -153,7 +158,9 @@ class Release: self.check_commit_release_ready() - def do(self, check_dirty: bool, check_branch: bool, with_release_branch: bool): + def do( + self, check_dirty: bool, check_branch: bool, with_release_branch: bool + ) -> None: self.check_prerequisites() if check_dirty: @@ -310,7 +317,7 @@ class Release: return self._version @version.setter - def version(self, version: ClickHouseVersion): + def version(self, version: ClickHouseVersion) -> None: if not isinstance(version, ClickHouseVersion): raise ValueError(f"version must be ClickHouseVersion, not {type(version)}") self._version = version @@ -320,7 +327,7 @@ class Release: return self._release_branch @release_branch.setter - def release_branch(self, branch: str): + def release_branch(self, branch: str) -> None: self._release_branch = release_branch(branch) @property @@ -328,7 +335,7 @@ class Release: return self._release_commit @release_commit.setter - def release_commit(self, release_commit: str): + def release_commit(self, release_commit: str) -> None: self._release_commit = commit(release_commit) @contextmanager @@ -367,7 +374,7 @@ class Release: yield @contextmanager - def _bump_testing_version(self, helper_branch: str): + def _bump_testing_version(self, helper_branch: str) -> Iterator[None]: self.read_version() self.version = self.version.update(self.release_type) self.version.with_description(VersionType.TESTING) @@ -387,7 +394,7 @@ class Release: yield @contextmanager - def _checkout(self, ref: str, with_checkout_back: bool = False): + def _checkout(self, ref: str, with_checkout_back: bool = False) -> Iterator[None]: orig_ref = self._git.branch or self._git.sha need_rollback = False if ref not in (self._git.branch, self._git.sha): @@ -406,7 +413,7 @@ class Release: self.run(rollback_cmd) @contextmanager - def _create_branch(self, name: str, start_point: str = ""): + def _create_branch(self, name: str, start_point: str = "") -> Iterator[None]: self.run(f"git branch {name} {start_point}") rollback_cmd = f"git branch -D {name}" self._rollback_stack.append(rollback_cmd) @@ -418,7 +425,7 @@ class Release: raise @contextmanager - def _create_gh_label(self, label: str, color_hex: str): + def _create_gh_label(self, label: str, color_hex: str) -> Iterator[None]: # API call, https://docs.github.com/en/rest/reference/issues#create-a-label self.run( f"gh api repos/{self.repo}/labels -f name={label} -f color={color_hex}" @@ -433,7 +440,7 @@ class Release: raise @contextmanager - def _create_gh_release(self, as_prerelease: bool): + def _create_gh_release(self, as_prerelease: bool) -> Iterator[None]: with self._create_tag(): # Preserve tag if version is changed tag = self.version.describe @@ -468,7 +475,9 @@ class Release: raise @contextmanager - def _push(self, ref: str, with_rollback_on_fail: bool = True, remote_ref: str = ""): + def _push( + self, ref: str, with_rollback_on_fail: bool = True, remote_ref: str = "" + ) -> Iterator[None]: if remote_ref == "": remote_ref = ref diff --git a/tests/ci/report.py b/tests/ci/report.py index a6700f50dfc..2904a5519a9 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -101,7 +101,7 @@ def _format_header(header, branch_name, branch_url=None): result = "ClickHouse " + result result += " for " if branch_url: - result += '{name}'.format(url=branch_url, name=branch_name) + result += f'{branch_name}' else: result += branch_name return result @@ -140,9 +140,7 @@ def _get_html_url(url): if isinstance(url, tuple): href, name = url[0], _get_html_url_name(url) if href and name: - return '{name}'.format( - href=href, name=_get_html_url_name(url) - ) + return f'{_get_html_url_name(url)}' return "" @@ -199,13 +197,7 @@ def create_test_html_report( num_fails = num_fails + 1 is_fail_id = 'id="fail' + str(num_fails) + '" ' - row += ( - "'.format(style) - + test_status - + "" - ) + row += f'{test_status}' if test_time is not None: row += "" + test_time + "" @@ -229,8 +221,8 @@ def create_test_html_report( if has_test_logs and not with_raw_logs: headers.append("Logs") - headers = "".join(["" + h + "" for h in headers]) - test_part = HTML_TEST_PART.format(headers=headers, rows=rows_part) + headers_html = "".join(["" + h + "" for h in headers]) + test_part = HTML_TEST_PART.format(headers=headers_html, rows=rows_part) else: test_part = "" @@ -317,33 +309,33 @@ def create_build_html_report( build_results, build_logs_urls, artifact_urls_list ): row = "" - row += "{}".format(build_result.compiler) + row += f"{build_result.compiler}" if build_result.build_type: - row += "{}".format(build_result.build_type) + row += f"{build_result.build_type}" else: - row += "{}".format("relwithdebuginfo") + row += "relwithdebuginfo" if build_result.sanitizer: - row += "{}".format(build_result.sanitizer) + row += f"{build_result.sanitizer}" else: - row += "{}".format("none") + row += "none" - row += "{}".format(build_result.libraries) + row += f"{build_result.libraries}" if build_result.status: style = _get_status_style(build_result.status) - row += '{}'.format(style, build_result.status) + row += f'{build_result.status}' else: style = _get_status_style("error") - row += '{}'.format(style, "error") + row += f'error' - row += 'link'.format(build_log_url) + row += f'link' if build_result.elapsed_seconds: delta = datetime.timedelta(seconds=build_result.elapsed_seconds) else: - delta = "unknown" + delta = "unknown" # type: ignore - row += "{}".format(str(delta)) + row += f"{delta}" links = "" link_separator = "
      " @@ -355,7 +347,7 @@ def create_build_html_report( links += link_separator if links: links = links[: -len(link_separator)] - row += "{}".format(links) + row += f"{links}" row += "" rows += row diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 39dbc938c8f..7119f443719 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -112,7 +112,7 @@ def should_run_checks_for_pr(pr_info: PRInfo) -> Tuple[bool, str, str]: return True, "No special conditions apply", "pending" -def check_pr_description(pr_info) -> Tuple[str, str]: +def check_pr_description(pr_info: PRInfo) -> Tuple[str, str]: lines = list( map(lambda x: x.strip(), pr_info.body.split("\n") if pr_info.body else []) ) diff --git a/tests/ci/runner_token_rotation_lambda/__init__.py b/tests/ci/runner_token_rotation_lambda/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ci/token_lambda/app.py b/tests/ci/runner_token_rotation_lambda/app.py similarity index 98% rename from tests/ci/token_lambda/app.py rename to tests/ci/runner_token_rotation_lambda/app.py index b8e54ed4e8d..70ee5da01f4 100644 --- a/tests/ci/token_lambda/app.py +++ b/tests/ci/runner_token_rotation_lambda/app.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -import requests import argparse -import jwt import sys import json import time +import boto3 # type: ignore +import jwt +import requests # type: ignore + def get_installation_id(jwt_token): headers = { @@ -51,8 +53,6 @@ def get_runner_registration_token(access_token): def get_key_and_app_from_aws(): - import boto3 - secret_name = "clickhouse_github_secret_key" session = boto3.session.Session() client = session.client( diff --git a/tests/ci/runner_token_rotation_lambda/build_and_deploy_archive.sh b/tests/ci/runner_token_rotation_lambda/build_and_deploy_archive.sh new file mode 120000 index 00000000000..96ba3fa024e --- /dev/null +++ b/tests/ci/runner_token_rotation_lambda/build_and_deploy_archive.sh @@ -0,0 +1 @@ +../team_keys_lambda/build_and_deploy_archive.sh \ No newline at end of file diff --git a/tests/ci/termination_lambda/requirements.txt b/tests/ci/runner_token_rotation_lambda/requirements.txt similarity index 100% rename from tests/ci/termination_lambda/requirements.txt rename to tests/ci/runner_token_rotation_lambda/requirements.txt diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index 24ff013d69a..03e855a0057 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -46,7 +46,7 @@ class S3Helper: self.host = host self.download_host = download_host - def _upload_file_to_s3(self, bucket_name, file_path, s3_path): + def _upload_file_to_s3(self, bucket_name: str, file_path: str, s3_path: str) -> str: logging.debug( "Start uploading %s to bucket=%s path=%s", file_path, bucket_name, s3_path ) @@ -110,7 +110,7 @@ class S3Helper: url = f"{self.download_host}/{bucket_name}/{s3_path}" return url.replace("+", "%2B").replace(" ", "%20") - def upload_test_report_to_s3(self, file_path, s3_path): + def upload_test_report_to_s3(self, file_path: str, s3_path: str) -> str: if CI: return self._upload_file_to_s3(S3_TEST_REPORTS_BUCKET, file_path, s3_path) else: @@ -296,7 +296,7 @@ class S3Helper: return False @staticmethod - def copy_file_to_local(bucket_name, file_path, s3_path): + def copy_file_to_local(bucket_name: str, file_path: str, s3_path: str) -> str: local_path = os.path.abspath( os.path.join(RUNNER_TEMP, "s3", bucket_name, s3_path) ) diff --git a/tests/ci/sqlancer_check.py b/tests/ci/sqlancer_check.py index 63c7d18fe46..5e94969d4b1 100644 --- a/tests/ci/sqlancer_check.py +++ b/tests/ci/sqlancer_check.py @@ -4,6 +4,7 @@ import logging import subprocess import os import sys +from typing import List, Tuple from github import Github @@ -137,7 +138,7 @@ if __name__ == "__main__": report_url = GITHUB_RUN_URL status = "success" - test_results = [] + test_results = [] # type: List[Tuple[str, str]] # Try to get status message saved by the SQLancer try: # with open( @@ -145,7 +146,7 @@ if __name__ == "__main__": # ) as status_f: # status = status_f.readline().rstrip("\n") if os.path.exists(os.path.join(workspace_path, "server_crashed.log")): - test_results.append("Server crashed", "FAIL") + test_results.append(("Server crashed", "FAIL")) with open( os.path.join(workspace_path, "summary.tsv"), "r", encoding="utf-8" ) as summary_f: diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 8f310eaa99d..c02128d114f 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -5,6 +5,7 @@ import logging import subprocess import os import sys +from typing import List, Tuple from github import Github @@ -44,8 +45,10 @@ def get_run_command( return cmd -def process_results(result_folder, server_log_path, run_log_path): - test_results = [] +def process_results( + result_folder: str, server_log_path: str, run_log_path: str +) -> Tuple[str, str, List[Tuple[str, str]], List[str]]: + test_results = [] # type: List[Tuple[str, str]] additional_files = [] # Just upload all files from result_folder. # If task provides processed results, then it's responsible for content @@ -89,7 +92,7 @@ def process_results(result_folder, server_log_path, run_log_path): results_path = os.path.join(result_folder, "test_results.tsv") with open(results_path, "r", encoding="utf-8") as results_file: - test_results = list(csv.reader(results_file, delimiter="\t")) + test_results = list(csv.reader(results_file, delimiter="\t")) # type: ignore if len(test_results) == 0: raise Exception("Empty results") diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 23a1dd467d7..70bf1cd4d17 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import argparse +import atexit import csv import logging import os import subprocess import sys -import atexit + +from typing import List, Tuple from clickhouse_helper import ( @@ -28,9 +30,18 @@ from upload_result_helper import upload_results NAME = "Style Check" +GIT_PREFIX = ( # All commits to remote are done as robot-clickhouse + "git -c user.email=robot-clickhouse@users.noreply.github.com " + "-c user.name=robot-clickhouse -c commit.gpgsign=false " + "-c core.sshCommand=" + "'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'" +) -def process_result(result_folder): - test_results = [] + +def process_result( + result_folder: str, +) -> Tuple[str, str, List[Tuple[str, str]], List[str]]: + test_results = [] # type: List[Tuple[str, str]] additional_files = [] # Just upload all files from result_folder. # If task provides processed results, then it's responsible @@ -57,7 +68,7 @@ def process_result(result_folder): try: results_path = os.path.join(result_folder, "test_results.tsv") with open(results_path, "r", encoding="utf-8") as fd: - test_results = list(csv.reader(fd, delimiter="\t")) + test_results = list(csv.reader(fd, delimiter="\t")) # type: ignore if len(test_results) == 0: raise Exception("Empty results") @@ -81,7 +92,7 @@ def parse_args(): return parser.parse_args() -def checkout_head(pr_info: PRInfo): +def checkout_head(pr_info: PRInfo) -> None: # It works ONLY for PRs, and only over ssh, so either # ROBOT_CLICKHOUSE_SSH_KEY should be set or ssh-agent should work assert pr_info.number @@ -89,14 +100,8 @@ def checkout_head(pr_info: PRInfo): # We can't push to forks, sorry folks return remote_url = pr_info.event["pull_request"]["base"]["repo"]["ssh_url"] - git_prefix = ( # All commits to remote are done as robot-clickhouse - "git -c user.email=robot-clickhouse@clickhouse.com " - "-c user.name=robot-clickhouse -c commit.gpgsign=false " - "-c core.sshCommand=" - "'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'" - ) fetch_cmd = ( - f"{git_prefix} fetch --depth=1 " + f"{GIT_PREFIX} fetch --depth=1 " f"{remote_url} {pr_info.head_ref}:head-{pr_info.head_ref}" ) if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""): @@ -107,7 +112,7 @@ def checkout_head(pr_info: PRInfo): git_runner(f"git checkout -f head-{pr_info.head_ref}") -def commit_push_staged(pr_info: PRInfo): +def commit_push_staged(pr_info: PRInfo) -> None: # It works ONLY for PRs, and only over ssh, so either # ROBOT_CLICKHOUSE_SSH_KEY should be set or ssh-agent should work assert pr_info.number @@ -118,15 +123,9 @@ def commit_push_staged(pr_info: PRInfo): if not git_staged: return remote_url = pr_info.event["pull_request"]["base"]["repo"]["ssh_url"] - git_prefix = ( # All commits to remote are done as robot-clickhouse - "git -c user.email=robot-clickhouse@clickhouse.com " - "-c user.name=robot-clickhouse -c commit.gpgsign=false " - "-c core.sshCommand=" - "'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'" - ) - git_runner(f"{git_prefix} commit -m 'Automatic style fix'") + git_runner(f"{GIT_PREFIX} commit -m 'Automatic style fix'") push_cmd = ( - f"{git_prefix} push {remote_url} head-{pr_info.head_ref}:{pr_info.head_ref}" + f"{GIT_PREFIX} push {remote_url} head-{pr_info.head_ref}:{pr_info.head_ref}" ) if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""): with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): diff --git a/tests/ci/team_keys_lambda/__init__.py b/tests/ci/team_keys_lambda/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ci/team_keys_lambda/app.py b/tests/ci/team_keys_lambda/app.py index 9e73a3f0993..870d41c441e 100644 --- a/tests/ci/team_keys_lambda/app.py +++ b/tests/ci/team_keys_lambda/app.py @@ -14,7 +14,7 @@ import boto3 # type: ignore class Keys(set): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.updated_at = 0 + self.updated_at = 0.0 def update_now(self): self.updated_at = datetime.now().timestamp() @@ -88,7 +88,7 @@ def get_token_from_aws() -> str: ) get_secret_value_response = client.get_secret_value(SecretId=secret_name) data = json.loads(get_secret_value_response["SecretString"]) - return data["clickhouse_robot_token"] + return data["clickhouse_robot_token"] # type: ignore def main(token: str, org: str, team_slug: str) -> str: diff --git a/tests/ci/team_keys_lambda/build_and_deploy_archive.sh b/tests/ci/team_keys_lambda/build_and_deploy_archive.sh index defa400453f..1ea2935c445 100644 --- a/tests/ci/team_keys_lambda/build_and_deploy_archive.sh +++ b/tests/ci/team_keys_lambda/build_and_deploy_archive.sh @@ -2,10 +2,13 @@ set -xeo pipefail WORKDIR=$(dirname "$0") +WORKDIR=$(readlink -f "${WORKDIR}") cd "$WORKDIR" -PY_EXEC=python3.9 -LAMBDA_NAME=$(basename "$PWD") +PY_VERSION=3.9 +PY_EXEC="python${PY_VERSION}" +DOCKER_IMAGE="python:${PY_VERSION}-slim" +LAMBDA_NAME=$(basename "$WORKDIR") LAMBDA_NAME=${LAMBDA_NAME//_/-} PACKAGE=lambda-package rm -rf "$PACKAGE" "$PACKAGE".zip @@ -14,10 +17,12 @@ cp app.py "$PACKAGE" if [ -f requirements.txt ]; then VENV=lambda-venv rm -rf "$VENV" lambda-package.zip - "$PY_EXEC" -m venv "$VENV" - # shellcheck disable=SC1091 - source "$VENV/bin/activate" - pip install -r requirements.txt + docker run --rm --user="${UID}" --volume="${WORKDIR}:/lambda" --workdir="/lambda" "${DOCKER_IMAGE}" \ + /bin/bash -c " + '$PY_EXEC' -m venv '$VENV' && + source '$VENV/bin/activate' && + pip install -r requirements.txt + " cp -rT "$VENV/lib/$PY_EXEC/site-packages/" "$PACKAGE" rm -r "$PACKAGE"/{pip,pip-*,setuptools,setuptools-*} fi diff --git a/tests/ci/tee_popen.py b/tests/ci/tee_popen.py index 7270cd6fb03..61404847bff 100644 --- a/tests/ci/tee_popen.py +++ b/tests/ci/tee_popen.py @@ -3,6 +3,7 @@ from subprocess import Popen, PIPE, STDOUT from threading import Thread from time import sleep +from typing import Optional import logging import os import sys @@ -18,7 +19,7 @@ class TeePopen: self.command = command self.log_file = log_file self.env = env - self.process = None + self._process = None # type: Optional[Popen] self.timeout = timeout def _check_timeout(self): @@ -51,7 +52,7 @@ class TeePopen: return self def __exit__(self, t, value, traceback): - for line in self.process.stdout: + for line in self.process.stdout: # type: ignore sys.stdout.write(line) self.log_file.write(line) @@ -59,8 +60,18 @@ class TeePopen: self.log_file.close() def wait(self): - for line in self.process.stdout: + for line in self.process.stdout: # type: ignore sys.stdout.write(line) self.log_file.write(line) return self.process.wait() + + @property + def process(self) -> Popen: + if self._process is not None: + return self._process + raise AttributeError("process is not created yet") + + @process.setter + def process(self, process: Popen) -> None: + self._process = process diff --git a/tests/ci/terminate_runner_lambda/__init__.py b/tests/ci/terminate_runner_lambda/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ci/termination_lambda/app.py b/tests/ci/terminate_runner_lambda/app.py similarity index 81% rename from tests/ci/termination_lambda/app.py rename to tests/ci/terminate_runner_lambda/app.py index ac1c7ad8df1..223555ced74 100644 --- a/tests/ci/termination_lambda/app.py +++ b/tests/ci/terminate_runner_lambda/app.py @@ -1,17 +1,18 @@ #!/usr/bin/env python3 -import requests import argparse -import jwt import sys import json import time from collections import namedtuple +from typing import Any, Dict, List, Tuple + +import boto3 # type: ignore +import requests # type: ignore +import jwt -def get_key_and_app_from_aws(): - import boto3 - +def get_key_and_app_from_aws() -> Tuple[str, int]: secret_name = "clickhouse_github_secret_key" session = boto3.session.Session() client = session.client( @@ -22,7 +23,7 @@ def get_key_and_app_from_aws(): return data["clickhouse-app-key"], int(data["clickhouse-app-id"]) -def get_installation_id(jwt_token): +def get_installation_id(jwt_token: str) -> int: headers = { "Authorization": f"Bearer {jwt_token}", "Accept": "application/vnd.github.v3+json", @@ -33,10 +34,12 @@ def get_installation_id(jwt_token): for installation in data: if installation["account"]["login"] == "ClickHouse": installation_id = installation["id"] - return installation_id + break + + return installation_id # type: ignore -def get_access_token(jwt_token, installation_id): +def get_access_token(jwt_token: str, installation_id: int) -> str: headers = { "Authorization": f"Bearer {jwt_token}", "Accept": "application/vnd.github.v3+json", @@ -47,15 +50,16 @@ def get_access_token(jwt_token, installation_id): ) response.raise_for_status() data = response.json() - return data["token"] + return data["token"] # type: ignore RunnerDescription = namedtuple( "RunnerDescription", ["id", "name", "tags", "offline", "busy"] ) +RunnerDescriptions = List[RunnerDescription] -def list_runners(access_token): +def list_runners(access_token: str) -> RunnerDescriptions: headers = { "Authorization": f"token {access_token}", "Accept": "application/vnd.github.v3+json", @@ -94,54 +98,9 @@ def list_runners(access_token): return result -def push_metrics_to_cloudwatch(listed_runners, namespace): - import boto3 - - client = boto3.client("cloudwatch") - metrics_data = [] - busy_runners = sum(1 for runner in listed_runners if runner.busy) - metrics_data.append( - { - "MetricName": "BusyRunners", - "Value": busy_runners, - "Unit": "Count", - } - ) - total_active_runners = sum(1 for runner in listed_runners if not runner.offline) - metrics_data.append( - { - "MetricName": "ActiveRunners", - "Value": total_active_runners, - "Unit": "Count", - } - ) - total_runners = len(listed_runners) - metrics_data.append( - { - "MetricName": "TotalRunners", - "Value": total_runners, - "Unit": "Count", - } - ) - if total_active_runners == 0: - busy_ratio = 100 - else: - busy_ratio = busy_runners / total_active_runners * 100 - - metrics_data.append( - { - "MetricName": "BusyRunnersRatio", - "Value": busy_ratio, - "Unit": "Percent", - } - ) - - client.put_metric_data(Namespace="RunnersMetrics", MetricData=metrics_data) - - -def how_many_instances_to_kill(event_data): +def how_many_instances_to_kill(event_data: dict) -> Dict[str, int]: data_array = event_data["CapacityToTerminate"] - to_kill_by_zone = {} + to_kill_by_zone = {} # type: Dict[str, int] for av_zone in data_array: zone_name = av_zone["AvailabilityZone"] to_kill = av_zone["Capacity"] @@ -149,15 +108,16 @@ def how_many_instances_to_kill(event_data): to_kill_by_zone[zone_name] = 0 to_kill_by_zone[zone_name] += to_kill + return to_kill_by_zone -def get_candidates_to_be_killed(event_data): +def get_candidates_to_be_killed(event_data: dict) -> Dict[str, List[str]]: data_array = event_data["Instances"] - instances_by_zone = {} + instances_by_zone = {} # type: Dict[str, List[str]] for instance in data_array: zone_name = instance["AvailabilityZone"] - instance_id = instance["InstanceId"] + instance_id = instance["InstanceId"] # type: str if zone_name not in instances_by_zone: instances_by_zone[zone_name] = [] instances_by_zone[zone_name].append(instance_id) @@ -165,7 +125,7 @@ def get_candidates_to_be_killed(event_data): return instances_by_zone -def delete_runner(access_token, runner): +def delete_runner(access_token: str, runner: RunnerDescription) -> bool: headers = { "Authorization": f"token {access_token}", "Accept": "application/vnd.github.v3+json", @@ -179,10 +139,12 @@ def delete_runner(access_token, runner): print( f"Response code deleting {runner.name} with id {runner.id} is {response.status_code}" ) - return response.status_code == 204 + return bool(response.status_code == 204) -def main(github_secret_key, github_app_id, event): +def main( + github_secret_key: str, github_app_id: int, event: dict +) -> Dict[str, List[str]]: print("Got event", json.dumps(event, sort_keys=True, indent=4)) to_kill_by_zone = how_many_instances_to_kill(event) instances_by_zone = get_candidates_to_be_killed(event) @@ -201,17 +163,16 @@ def main(github_secret_key, github_app_id, event): to_delete_runners = [] instances_to_kill = [] - for zone in to_kill_by_zone: - num_to_kill = to_kill_by_zone[zone] + for zone, num_to_kill in to_kill_by_zone.items(): candidates = instances_by_zone[zone] if num_to_kill > len(candidates): raise Exception( f"Required to kill {num_to_kill}, but have only {len(candidates)} candidates in AV {zone}" ) - delete_for_av = [] + delete_for_av = [] # type: RunnerDescriptions for candidate in candidates: - if candidate not in set([runner.name for runner in runners]): + if candidate not in set(runner.name for runner in runners): print( f"Candidate {candidate} was not in runners list, simply delete it" ) @@ -254,16 +215,12 @@ def main(github_secret_key, github_app_id, event): else: print(f"Cannot delete {runner.name} from github") - ## push metrics - # runners = list_runners(access_token) - # push_metrics_to_cloudwatch(runners, 'RunnersMetrics') - response = {"InstanceIDs": instances_to_kill} print(response) return response -def handler(event, context): +def handler(event: dict, context: Any) -> Dict[str, List[str]]: private_key, app_id = get_key_and_app_from_aws() return main(private_key, app_id, event) diff --git a/tests/ci/terminate_runner_lambda/build_and_deploy_archive.sh b/tests/ci/terminate_runner_lambda/build_and_deploy_archive.sh new file mode 120000 index 00000000000..96ba3fa024e --- /dev/null +++ b/tests/ci/terminate_runner_lambda/build_and_deploy_archive.sh @@ -0,0 +1 @@ +../team_keys_lambda/build_and_deploy_archive.sh \ No newline at end of file diff --git a/tests/ci/token_lambda/requirements.txt b/tests/ci/terminate_runner_lambda/requirements.txt similarity index 100% rename from tests/ci/token_lambda/requirements.txt rename to tests/ci/terminate_runner_lambda/requirements.txt diff --git a/tests/ci/termination_lambda/Dockerfile b/tests/ci/termination_lambda/Dockerfile deleted file mode 100644 index 0d50224c51d..00000000000 --- a/tests/ci/termination_lambda/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM public.ecr.aws/lambda/python:3.9 - -# Install the function's dependencies using file requirements.txt -# from your project folder. - -COPY requirements.txt . -RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}" - -# Copy function code -COPY app.py ${LAMBDA_TASK_ROOT} - -# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) -CMD [ "app.handler" ] diff --git a/tests/ci/token_lambda/Dockerfile b/tests/ci/token_lambda/Dockerfile deleted file mode 100644 index 0d50224c51d..00000000000 --- a/tests/ci/token_lambda/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM public.ecr.aws/lambda/python:3.9 - -# Install the function's dependencies using file requirements.txt -# from your project folder. - -COPY requirements.txt . -RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}" - -# Copy function code -COPY app.py ${LAMBDA_TASK_ROOT} - -# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) -CMD [ "app.handler" ] diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index c2dfab9dddc..4777296da18 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -5,6 +5,7 @@ import os import sys import subprocess import atexit +from typing import List, Tuple from github import Github @@ -37,14 +38,16 @@ def get_test_name(line): raise Exception(f"No test name in line '{line}'") -def process_result(result_folder): +def process_results( + result_folder: str, +) -> Tuple[str, str, List[Tuple[str, str]], List[str]]: OK_SIGN = "OK ]" FAILED_SIGN = "FAILED ]" SEGFAULT = "Segmentation fault" SIGNAL = "received signal SIG" PASSED = "PASSED" - summary = [] + summary = [] # type: List[Tuple[str, str]] total_counter = 0 failed_counter = 0 result_log_path = f"{result_folder}/test_result.txt" @@ -151,7 +154,7 @@ if __name__ == "__main__": subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) s3_helper = S3Helper() - state, description, test_results, additional_logs = process_result(test_output) + state, description, test_results, additional_logs = process_results(test_output) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) diff --git a/tests/ci/upload_result_helper.py b/tests/ci/upload_result_helper.py index e145df02f80..9fcd3733acb 100644 --- a/tests/ci/upload_result_helper.py +++ b/tests/ci/upload_result_helper.py @@ -16,7 +16,7 @@ def process_logs( ): logging.info("Upload files to s3 %s", additional_logs) - processed_logs = {} + processed_logs = {} # type: ignore # Firstly convert paths of logs from test_results to urls to s3. for test_result in test_results: if len(test_result) <= 3 or with_raw_logs: diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 162bab6a50a..69cfba64be3 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -2,9 +2,9 @@ import logging import os.path as p from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Literal, Optional, Tuple, Union -from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix +from git_helper import TWEAK, Git as Git, get_tags, git_runner, removeprefix FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt" CHANGELOG_IN_PATH = "debian/changelog.in" @@ -45,7 +45,7 @@ class ClickHouseVersion: patch: Union[int, str], revision: Union[int, str], git: Optional[Git], - tweak: str = None, + tweak: Optional[str] = None, ): self._major = int(major) self._minor = int(minor) @@ -59,10 +59,15 @@ class ClickHouseVersion: self._tweak = self._git.tweak self._describe = "" - def update(self, part: str) -> "ClickHouseVersion": + def update(self, part: Literal["major", "minor", "patch"]) -> "ClickHouseVersion": """If part is valid, returns a new version""" - method = getattr(self, f"{part}_update") - return method() + if part == "major": + return self.major_update() + if part == "minor": + return self.minor_update() + if part == "patch": + return self.patch_update() + raise KeyError(f"wrong part {part} is used") def major_update(self) -> "ClickHouseVersion": if self._git is not None: @@ -139,10 +144,10 @@ class ClickHouseVersion: raise ValueError(f"version type {version_type} not in {VersionType.VALID}") self._describe = f"v{self.string}-{version_type}" - def __eq__(self, other) -> bool: + def __eq__(self, other: Any) -> bool: if not isinstance(self, type(other)): return NotImplemented - return ( + return bool( self.major == other.major and self.minor == other.minor and self.patch == other.patch @@ -170,7 +175,7 @@ class VersionType: VALID = (TESTING, PRESTABLE, STABLE, LTS) -def validate_version(version: str): +def validate_version(version: str) -> None: parts = version.split(".") if len(parts) != 4: raise ValueError(f"{version} does not contain 4 parts") @@ -259,7 +264,7 @@ def get_tagged_versions() -> List[ClickHouseVersion]: def update_cmake_version( version: ClickHouseVersion, versions_path: str = FILE_WITH_VERSION_PATH, -): +) -> None: path_to_file = get_abs_path(versions_path) with open(path_to_file, "w", encoding="utf-8") as f: f.write(VERSIONS_TEMPLATE.format_map(version.as_dict())) @@ -269,7 +274,7 @@ def update_contributors( relative_contributors_path: str = GENERATED_CONTRIBUTORS, force: bool = False, raise_error: bool = False, -): +) -> None: # Check if we have shallow checkout by comparing number of lines # '--is-shallow-repository' is in git since 2.15, 2017-10-30 if git_runner.run("git rev-parse --is-shallow-repository") == "true" and not force: diff --git a/tests/ci/version_test.py b/tests/ci/version_test.py index 86a2d58c3c8..abd0f9349f4 100644 --- a/tests/ci/version_test.py +++ b/tests/ci/version_test.py @@ -17,9 +17,9 @@ class TestFunctions(unittest.TestCase): ("v1.1.1.2-testing", vh.get_version_from_string("1.1.1.2")), ("refs/tags/v1.1.1.2-testing", vh.get_version_from_string("1.1.1.2")), ) - for case in cases: - version = vh.version_arg(case[0]) - self.assertEqual(case[1], version) + for test_case in cases: + version = vh.version_arg(test_case[0]) + self.assertEqual(test_case[1], version) error_cases = ( "0.0.0", "1.1.1.a", @@ -28,6 +28,6 @@ class TestFunctions(unittest.TestCase): "v1.1.1.2-testin", "refs/tags/v1.1.1.2-testin", ) - for case in error_cases: + for error_case in error_cases: with self.assertRaises(ArgumentTypeError): - version = vh.version_arg(case[0]) + version = vh.version_arg(error_case[0]) diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh index 66a38a6a37d..64f11b41777 100644 --- a/tests/ci/worker/init_runner.sh +++ b/tests/ci/worker/init_runner.sh @@ -46,15 +46,17 @@ curl "${TEAM_KEYS_URL}" > /home/ubuntu/.ssh/authorized_keys2 chown ubuntu: /home/ubuntu/.ssh -R -# Create a pre-run script that will restart docker daemon before the job started +# Create a pre-run script that will provide diagnostics info mkdir -p /tmp/actions-hooks -cat > /tmp/actions-hooks/pre-run.sh << 'EOF' +cat > /tmp/actions-hooks/pre-run.sh << EOF #!/bin/bash -set -xuo pipefail +set -uo pipefail echo "Runner's public DNS: $(ec2metadata --public-hostname)" +echo "Runner's labels: ${LABELS}" EOF +# Create a post-run script that will restart docker daemon before the job started cat > /tmp/actions-hooks/post-run.sh << 'EOF' #!/bin/bash set -xuo pipefail diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py index 23e808b0861..d285e29943d 100644 --- a/tests/ci/workflow_approve_rerun_lambda/app.py +++ b/tests/ci/workflow_approve_rerun_lambda/app.py @@ -313,7 +313,7 @@ def check_suspicious_changed_files(changed_files): return False -def approve_run(workflow_description: WorkflowDescription, token): +def approve_run(workflow_description: WorkflowDescription, token: str) -> None: url = f"{workflow_description.api_url}/approve" _exec_post_with_retry(url, token) @@ -391,7 +391,7 @@ def rerun_workflow(workflow_description, token): def check_workflow_completed( - event_data, workflow_description: WorkflowDescription, token: str + event_data: dict, workflow_description: WorkflowDescription, token: str ) -> bool: if workflow_description.action == "completed": attempt = 0 diff --git a/tests/config/config.d/clusters.xml b/tests/config/config.d/clusters.xml index 5d14bc7e980..9d58606c02f 100644 --- a/tests/config/config.d/clusters.xml +++ b/tests/config/config.d/clusters.xml @@ -57,5 +57,20 @@       + + + + 127.0.0.1 + 9000 + + + + + shard_1 + 127.0.0.2 + 9000 + + + diff --git a/tests/config/config.d/compressed_marks_and_index.xml b/tests/config/config.d/compressed_marks_and_index.xml new file mode 100644 index 00000000000..ba8bdfe9658 --- /dev/null +++ b/tests/config/config.d/compressed_marks_and_index.xml @@ -0,0 +1,6 @@ + + + true + true + + diff --git a/tests/config/install.sh b/tests/config/install.sh index 0c1908cc30d..bba01d8d2e4 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -53,6 +53,7 @@ ln -sf $SRC_PATH/config.d/nlp.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/enable_keeper_map.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/custom_disks_base_path.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/display_name.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/compressed_marks_and_index.xml $DEST_SERVER_PATH/config.d/ # Not supported with fasttest. if [ "${DEST_SERVER_PATH}" = "/etc/clickhouse-server" ] diff --git a/tests/integration/helpers/wait_for_helpers.py b/tests/integration/helpers/wait_for_helpers.py new file mode 100644 index 00000000000..b041de99282 --- /dev/null +++ b/tests/integration/helpers/wait_for_helpers.py @@ -0,0 +1,30 @@ +import time +from helpers.test_tools import assert_eq_with_retry + + +def _parse_table_database(table, database): + if database is not None: + return table, database + + if "." in table: + return reversed(table.split(".", 1)) + + return table, "default" + + +def wait_for_delete_inactive_parts(node, table, database=None, **kwargs): + table, database = _parse_table_database(table, database) + inactive_parts_query = ( + f"SELECT count() FROM system.parts " + f"WHERE not active AND table = '{table}' AND database = '{database}';" + ) + assert_eq_with_retry(node, inactive_parts_query, "0\n", **kwargs) + + +def wait_for_delete_empty_parts(node, table, database=None, **kwargs): + table, database = _parse_table_database(table, database) + empty_parts_query = ( + f"SELECT count() FROM system.parts " + f"WHERE active AND rows = 0 AND table = '{table}' AND database = '{database}'" + ) + assert_eq_with_retry(node, empty_parts_query, "0\n", **kwargs) diff --git a/tests/integration/test_alter_settings_on_cluster/test.py b/tests/integration/test_alter_settings_on_cluster/test.py index 6ab3d446b59..32f7f2efa30 100644 --- a/tests/integration/test_alter_settings_on_cluster/test.py +++ b/tests/integration/test_alter_settings_on_cluster/test.py @@ -52,3 +52,24 @@ def test_default_database_on_cluster(started_cluster): database="test_default_database", sql="SHOW CREATE test_local_table FORMAT TSV", ).endswith("old_parts_lifetime = 100\n") + + ch1.query_and_get_error( + database="test_default_database", + sql="ALTER TABLE test_local_table MODIFY SETTING temporary_directories_lifetime = 1 RESET SETTING old_parts_lifetime;", + ) + + ch1.query_and_get_error( + database="test_default_database", + sql="ALTER TABLE test_local_table RESET SETTING old_parts_lifetime MODIFY SETTING temporary_directories_lifetime = 1;", + ) + + ch1.query( + database="test_default_database", + sql="ALTER TABLE test_local_table ON CLUSTER 'cluster' RESET SETTING old_parts_lifetime;", + ) + + for node in [ch1, ch2]: + assert not node.query( + database="test_default_database", + sql="SHOW CREATE test_local_table FORMAT TSV", + ).endswith("old_parts_lifetime = 100\n") diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index c94dc6d4a87..7eeabde1380 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -1,6 +1,7 @@ import pytest import asyncio import re +import random import os.path from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry, TSV @@ -1158,3 +1159,71 @@ def test_mutation(): instance.query("DROP TABLE test.table") instance.query(f"RESTORE TABLE test.table FROM {backup_name}") + + +def test_tables_dependency(): + instance.query("CREATE DATABASE test") + instance.query("CREATE DATABASE test2") + + # For this test we use random names of tables to check they're created according to their dependency (not just in alphabetic order). + random_table_names = [f"{chr(ord('A')+i)}" for i in range(0, 10)] + random.shuffle(random_table_names) + random_table_names = [ + random.choice(["test", "test2"]) + "." + table_name + for table_name in random_table_names + ] + print(f"random_table_names={random_table_names}") + + t1 = random_table_names[0] + t2 = random_table_names[1] + t3 = random_table_names[2] + t4 = random_table_names[3] + t5 = random_table_names[4] + t6 = random_table_names[5] + + # Create a materialized view and a dictionary with a local table as source. + instance.query( + f"CREATE TABLE {t1} (x Int64, y String) ENGINE=MergeTree ORDER BY tuple()" + ) + + instance.query( + f"CREATE TABLE {t2} (x Int64, y String) ENGINE=MergeTree ORDER BY tuple()" + ) + + instance.query(f"CREATE MATERIALIZED VIEW {t3} TO {t2} AS SELECT x, y FROM {t1}") + + instance.query( + f"CREATE DICTIONARY {t4} (x Int64, y String) PRIMARY KEY x SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE '{t1.split('.')[1]}' DB '{t1.split('.')[0]}')) LAYOUT(FLAT()) LIFETIME(0)" + ) + + instance.query(f"CREATE TABLE {t5} AS dictionary({t4})") + + instance.query( + f"CREATE TABLE {t6}(x Int64, y String DEFAULT dictGet({t4}, 'y', x)) ENGINE=MergeTree ORDER BY tuple()" + ) + + # Make backup. + backup_name = new_backup_name() + instance.query(f"BACKUP DATABASE test, DATABASE test2 TO {backup_name}") + + # Drop everything in reversive order. + def drop(): + instance.query(f"DROP TABLE {t6} NO DELAY") + instance.query(f"DROP TABLE {t5} NO DELAY") + instance.query(f"DROP DICTIONARY {t4}") + instance.query(f"DROP TABLE {t3} NO DELAY") + instance.query(f"DROP TABLE {t2} NO DELAY") + instance.query(f"DROP TABLE {t1} NO DELAY") + instance.query("DROP DATABASE test NO DELAY") + instance.query("DROP DATABASE test2 NO DELAY") + + drop() + + # Restore everything and check. + instance.query(f"RESTORE ALL FROM {backup_name}") + + assert instance.query( + "SELECT concat(database, '.', name) AS c FROM system.tables WHERE database IN ['test', 'test2'] ORDER BY c" + ) == TSV(sorted([t1, t2, t3, t4, t5, t6])) + + drop() diff --git a/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml b/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml index 04d34327fef..c823dd02d5a 100644 --- a/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml +++ b/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml @@ -1,5 +1,7 @@ 0 + 0 + 0 diff --git a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py new file mode 100644 index 00000000000..94c788f8f91 --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py @@ -0,0 +1,85 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance( + "node1", + with_zookeeper=False, + image="yandex/clickhouse-server", + tag="21.1", + stay_alive=True, + with_installed_binary=True, +) +node2 = cluster.add_instance( + "node2", + with_zookeeper=False, + image="yandex/clickhouse-server", + tag="21.1", + stay_alive=True, + with_installed_binary=True, +) +node3 = cluster.add_instance("node3", with_zookeeper=False) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_backward_compatability(start_cluster): + node1.query("create table t (a UInt64) engine = MergeTree order by a") + node2.query("create table t (a UInt64) engine = MergeTree order by a") + node3.query("create table t (a UInt64) engine = MergeTree order by a") + + node1.query("insert into t select number % 100000 from numbers_mt(1000000)") + node2.query("insert into t select number % 100000 from numbers_mt(1000000)") + node3.query("insert into t select number % 100000 from numbers_mt(1000000)") + + assert ( + node1.query( + """ + select count() + from remote('node{1,2,3}', default, t) + group by a + limit 1 offset 12345 + settings optimize_aggregation_in_order = 1 + """ + ) + == "30\n" + ) + + assert ( + node2.query( + """ + select count() + from remote('node{1,2,3}', default, t) + group by a + limit 1 offset 12345 + settings optimize_aggregation_in_order = 1 + """ + ) + == "30\n" + ) + + assert ( + node3.query( + """ + select count() + from remote('node{1,2,3}', default, t) + group by a + limit 1 offset 12345 + settings optimize_aggregation_in_order = 1 + """ + ) + == "30\n" + ) + + node1.query("drop table t") + node2.query("drop table t") + node3.query("drop table t") diff --git a/tests/integration/test_broken_detached_part_clean_up/test.py b/tests/integration/test_broken_detached_part_clean_up/test.py index e9ef0067ca5..d39946102ef 100644 --- a/tests/integration/test_broken_detached_part_clean_up/test.py +++ b/tests/integration/test_broken_detached_part_clean_up/test.py @@ -280,6 +280,7 @@ def test_store_cleanup(started_cluster): "Removing unused directory", timeout=90, look_behind_lines=1000 ) node1.wait_for_log_line("directories from store") + node1.wait_for_log_line("Nothing to clean up from store/") store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) assert "100" in store diff --git a/tests/integration/test_detached_parts_metrics/test.py b/tests/integration/test_detached_parts_metrics/test.py index 62b70ebd430..fb312f8d224 100644 --- a/tests/integration/test_detached_parts_metrics/test.py +++ b/tests/integration/test_detached_parts_metrics/test.py @@ -2,6 +2,8 @@ import time import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry +from helpers.wait_for_helpers import wait_for_delete_inactive_parts +from helpers.wait_for_helpers import wait_for_delete_empty_parts cluster = ClickHouseCluster(__file__) @@ -20,7 +22,7 @@ def started_cluster(): cluster.shutdown() -def test_event_time_microseconds_field(started_cluster): +def test_numbers_of_detached_parts(started_cluster): cluster.start() query_create = """ CREATE TABLE t @@ -68,6 +70,7 @@ def test_event_time_microseconds_field(started_cluster): # detach some parts and wait until asynchronous metrics notice it node1.query("ALTER TABLE t DETACH PARTITION '20220901';") + wait_for_delete_empty_parts(node1, "t") assert 2 == int(node1.query(query_count_detached_parts)) assert 1 == int(node1.query(query_count_active_parts)) @@ -81,6 +84,7 @@ def test_event_time_microseconds_field(started_cluster): # detach the rest parts and wait until asynchronous metrics notice it node1.query("ALTER TABLE t DETACH PARTITION ALL") + wait_for_delete_empty_parts(node1, "t") assert 3 == int(node1.query(query_count_detached_parts)) assert 0 == int(node1.query(query_count_active_parts)) diff --git a/tests/integration/test_drop_is_lock_free/__init__.py b/tests/integration/test_drop_is_lock_free/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_drop_is_lock_free/configs/keeper.xml b/tests/integration/test_drop_is_lock_free/configs/keeper.xml new file mode 100644 index 00000000000..f4fde78cc97 --- /dev/null +++ b/tests/integration/test_drop_is_lock_free/configs/keeper.xml @@ -0,0 +1,30 @@ + + + + localhost + 9181 + + + + + 9181 + 1 + + + 10000 + 30000 + false + 60000 + + 1000000000000000 + + + + + 1 + localhost + 9234 + + + + diff --git a/tests/integration/test_drop_is_lock_free/configs/transactions.xml b/tests/integration/test_drop_is_lock_free/configs/transactions.xml new file mode 100644 index 00000000000..a8d3e8fbf6d --- /dev/null +++ b/tests/integration/test_drop_is_lock_free/configs/transactions.xml @@ -0,0 +1,14 @@ + + 42 + + + 100500 + 0 + + + + system + transactions_info_log
      + 7500 +       +       diff --git a/tests/integration/test_drop_is_lock_free/test.py b/tests/integration/test_drop_is_lock_free/test.py new file mode 100644 index 00000000000..8d92d784226 --- /dev/null +++ b/tests/integration/test_drop_is_lock_free/test.py @@ -0,0 +1,222 @@ +import time +import pytest +import logging +from contextlib import contextmanager +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + stay_alive=True, + with_zookeeper=False, + main_configs=[ + "configs/keeper.xml", + "configs/transactions.xml", + ], +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(scope="function") +def test_name(request): + return request.node.name + + +@pytest.fixture(scope="function") +def exclusive_table(test_name): + normalized = ( + test_name.replace("[", "_") + .replace("]", "_") + .replace(" ", "_") + .replace("-", "_") + ) + return "table_" + normalized + + +def get_event_select_count(): + return int( + node.query( + """ + SELECT value FROM system.events WHERE event = 'SelectQuery'; + """ + ) + ) + + +def get_query_processes_count(query_id): + q = f""" + SELECT count() FROM system.processes WHERE query_id = '{query_id}'; + """ + return q + + +def is_query_running(query_id): + return 1 == int(node.query(get_query_processes_count(query_id))) + + +def wait_select_start(query_id): + assert_eq_with_retry( + node, + get_query_processes_count(query_id), + "1\n", + ) + + +LOCK_FREE_QUERIES = { + "detach table": "DETACH TABLE {table};", + "drop part": "ALTER TABLE {table} DROP PART 'all_1_1_0';", + "detach part": "ALTER TABLE {table} DETACH PART 'all_1_1_0';", + "truncate": "TRUNCATE TABLE {table};", +} + + +@pytest.mark.parametrize( + "lock_free_query", LOCK_FREE_QUERIES.values(), ids=LOCK_FREE_QUERIES.keys() +) +def test_query_is_lock_free(lock_free_query, exclusive_table): + node.query( + f""" + CREATE TABLE {exclusive_table} + (a Int64) + Engine=MergeTree ORDER BY a; + """ + ) + node.query( + f""" + INSERT INTO {exclusive_table} SELECT number FROM numbers(50); + """ + ) + + query_id = "select-" + exclusive_table + + select_handler = node.get_query_request( + f""" + SELECT sleepEachRow(3) FROM {exclusive_table}; + """, + query_id=query_id, + ) + wait_select_start(query_id) + + for _ in [1, 2, 3, 4, 5]: + assert is_query_running(query_id) + assert select_handler.process.poll() is None + time.sleep(1) + + node.query(lock_free_query.format(table=exclusive_table)) + + assert is_query_running(query_id) + + if "DETACH TABLE" in lock_free_query: + result = node.query_and_get_error( + f""" + SELECT count() FROM {exclusive_table}; + """ + ) + assert f"Table default.{exclusive_table} doesn't exist" in result + else: + assert 0 == int( + node.query( + f""" + SELECT count() FROM {exclusive_table}; + """ + ) + ) + + +PERMANENT_QUERIES = { + "truncate": ("TRUNCATE TABLE {table};", 0), + "detach-partition-all": ("ALTER TABLE {table} DETACH PARTITION ALL;", 0), + "detach-part": ("ALTER TABLE {table} DETACH PARTITION '20221001';", 49), + "drop-part": ("ALTER TABLE {table} DROP PART '20220901_1_1_0';", 49), +} + + +@pytest.mark.parametrize( + "transaction", ["NoTx", "TxCommit", "TxRollback", "TxNotFinished"] +) +@pytest.mark.parametrize( + "permanent", PERMANENT_QUERIES.values(), ids=PERMANENT_QUERIES.keys() +) +def test_query_is_permanent(transaction, permanent, exclusive_table): + node.query( + f""" + CREATE TABLE {exclusive_table} + ( + a Int64, + date Date + ) + Engine=MergeTree + PARTITION BY date + ORDER BY a; + """ + ) + node.query( + f""" + INSERT INTO {exclusive_table} SELECT number, toDate('2022-09-01') + INTERVAL number DAY FROM numbers(50); + """ + ) + + query_id = "select-" + exclusive_table + + select_handler = node.get_query_request( + f""" + SELECT sleepEachRow(3) FROM {exclusive_table}; + """, + query_id=query_id, + ) + wait_select_start(query_id) + + for _ in [1, 2, 3, 4, 5]: + assert is_query_running(query_id) + assert select_handler.process.poll() is None + time.sleep(1) + + permanent_query = permanent[0] + result = permanent[1] + statement = permanent_query.format(table=exclusive_table) + if transaction == "TxCommit": + query = f""" + BEGIN TRANSACTION; + {statement} + COMMIT; + """ + elif transaction == "TxRollback": + query = f""" + BEGIN TRANSACTION; + {statement} + ROLLBACK; + """ + result = 50 + elif transaction == "TxNotFinished": + query = f""" + BEGIN TRANSACTION; + {statement} + """ + result = 50 + else: + query = statement + + node.query(query) + + node.restart_clickhouse(kill=True) + + assert result == int( + node.query( + f""" + SELECT count() FROM {exclusive_table}; + """ + ) + ) diff --git a/tests/integration/test_hive_query/test.py b/tests/integration/test_hive_query/test.py index a498320ed5b..791ae03f9f6 100644 --- a/tests/integration/test_hive_query/test.py +++ b/tests/integration/test_hive_query/test.py @@ -1,8 +1,14 @@ +import pytest + +# FIXME This test is too flaky +# https://github.com/ClickHouse/ClickHouse/issues/43541 + +pytestmark = pytest.mark.skip + import logging import os import time -import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_keeper_four_word_command/test.py b/tests/integration/test_keeper_four_word_command/test.py index bc6e227e861..04f6800b92b 100644 --- a/tests/integration/test_keeper_four_word_command/test.py +++ b/tests/integration/test_keeper_four_word_command/test.py @@ -285,6 +285,8 @@ def test_cmd_conf(started_cluster): assert result["fresh_log_gap"] == "200" assert result["max_requests_batch_size"] == "100" + assert result["max_request_queue_size"] == "100000" + assert result["max_requests_quick_batch_size"] == "10" assert result["quorum_reads"] == "false" assert result["force_sync"] == "true" diff --git a/tests/integration/test_keeper_map/test.py b/tests/integration/test_keeper_map/test.py index 8f515077e8f..71f6343101a 100644 --- a/tests/integration/test_keeper_map/test.py +++ b/tests/integration/test_keeper_map/test.py @@ -5,7 +5,7 @@ import random from itertools import count from sys import stdout -from multiprocessing import Pool +from multiprocessing.dummy import Pool from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry, assert_logs_contain diff --git a/tests/integration/test_keeper_zookeeper_converter/test.py b/tests/integration/test_keeper_zookeeper_converter/test.py index af8d1ca4bf9..aa2e435ce36 100644 --- a/tests/integration/test_keeper_zookeeper_converter/test.py +++ b/tests/integration/test_keeper_zookeeper_converter/test.py @@ -2,14 +2,9 @@ import pytest from helpers.cluster import ClickHouseCluster import helpers.keeper_utils as keeper_utils -from kazoo.client import KazooClient, KazooState -from kazoo.security import ACL, make_digest_acl, make_acl -from kazoo.exceptions import ( - AuthFailedError, - InvalidACLError, - NoAuthError, - KazooException, -) +from kazoo.client import KazooClient +from kazoo.retry import KazooRetry +from kazoo.security import make_acl import os import time @@ -99,7 +94,9 @@ def get_fake_zk(timeout=60.0): def get_genuine_zk(timeout=60.0): _genuine_zk_instance = KazooClient( - hosts=cluster.get_instance_ip("node") + ":2181", timeout=timeout + hosts=cluster.get_instance_ip("node") + ":2181", + timeout=timeout, + connection_retry=KazooRetry(max_tries=20), ) _genuine_zk_instance.start() return _genuine_zk_instance @@ -225,6 +222,12 @@ def test_smoke(started_cluster, create_snapshots): compare_states(genuine_connection, fake_connection) + genuine_connection.stop() + genuine_connection.close() + + fake_connection.stop() + fake_connection.close() + def get_bytes(s): return s.encode() @@ -309,6 +312,12 @@ def test_simple_crud_requests(started_cluster, create_snapshots): second_children = list(sorted(fake_connection.get_children("/test_sequential"))) assert first_children == second_children, "Childrens are not equal on path " + path + genuine_connection.stop() + genuine_connection.close() + + fake_connection.stop() + fake_connection.close() + @pytest.mark.parametrize(("create_snapshots"), [True, False]) def test_multi_and_failed_requests(started_cluster, create_snapshots): @@ -379,6 +388,12 @@ def test_multi_and_failed_requests(started_cluster, create_snapshots): assert eph1 == eph2 compare_stats(stat1, stat2, "/test_multitransactions", ignore_pzxid=True) + genuine_connection.stop() + genuine_connection.close() + + fake_connection.stop() + fake_connection.close() + @pytest.mark.parametrize(("create_snapshots"), [True, False]) def test_acls(started_cluster, create_snapshots): @@ -446,3 +461,9 @@ def test_acls(started_cluster, create_snapshots): "user2:lo/iTtNMP+gEZlpUNaCqLYO3i5U=", "user3:wr5Y0kEs9nFX3bKrTMKxrlcFeWo=", ) + + genuine_connection.stop() + genuine_connection.close() + + fake_connection.stop() + fake_connection.close() diff --git a/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py index bed7772a3dd..5b75b0dfc38 100644 --- a/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py @@ -2184,3 +2184,44 @@ def savepoint(clickhouse_node, mysql_node, mysql_host): mysql_node.query(f"INSERT INTO {db}.t1 VALUES (2)") mysql_node.query("ROLLBACK TO savepoint_1") mysql_node.query("COMMIT") + + +def dropddl(clickhouse_node, mysql_node, mysql_host): + db = "dropddl" + clickhouse_node.query(f"DROP DATABASE IF EXISTS {db}") + mysql_node.query(f"DROP DATABASE IF EXISTS {db}") + mysql_node.query(f"CREATE DATABASE {db}") + mysql_node.query(f"CREATE TABLE {db}.t1 (a INT PRIMARY KEY, b INT)") + mysql_node.query(f"CREATE TABLE {db}.t2 (a INT PRIMARY KEY, b INT)") + mysql_node.query(f"CREATE TABLE {db}.t3 (a INT PRIMARY KEY, b INT)") + mysql_node.query(f"CREATE TABLE {db}.t4 (a INT PRIMARY KEY, b INT)") + mysql_node.query(f"CREATE VIEW {db}.v1 AS SELECT * FROM {db}.t1") + mysql_node.query(f"INSERT INTO {db}.t1(a, b) VALUES(1, 1)") + + clickhouse_node.query( + f"CREATE DATABASE {db} ENGINE = MaterializeMySQL('{mysql_host}:3306', '{db}', 'root', 'clickhouse')" + ) + check_query( + clickhouse_node, + f"SELECT count() FROM system.tables where database = '{db}' FORMAT TSV", + "4\n", + ) + check_query(clickhouse_node, f"SELECT * FROM {db}.t1 FORMAT TSV", "1\t1\n") + mysql_node.query(f"DROP EVENT IF EXISTS {db}.event_name") + mysql_node.query(f"DROP VIEW IF EXISTS {db}.view_name") + mysql_node.query(f"DROP FUNCTION IF EXISTS {db}.function_name") + mysql_node.query(f"DROP TRIGGER IF EXISTS {db}.trigger_name") + mysql_node.query(f"DROP INDEX `PRIMARY` ON {db}.t2") + mysql_node.query(f"DROP TABLE {db}.t3") + mysql_node.query(f"DROP TABLE if EXISTS {db}.t3,{db}.t4") + mysql_node.query(f"TRUNCATE TABLE {db}.t1") + mysql_node.query(f"INSERT INTO {db}.t2(a, b) VALUES(1, 1)") + check_query(clickhouse_node, f"SELECT * FROM {db}.t2 FORMAT TSV", "1\t1\n") + check_query(clickhouse_node, f"SELECT count() FROM {db}.t1 FORMAT TSV", "0\n") + check_query( + clickhouse_node, + f"SELECT name FROM system.tables where database = '{db}' FORMAT TSV", + "t1\nt2\n", + ) + mysql_node.query(f"DROP DATABASE {db}") + clickhouse_node.query(f"DROP DATABASE {db}") diff --git a/tests/integration/test_materialized_mysql_database/test.py b/tests/integration/test_materialized_mysql_database/test.py index 0e33c01a6c9..a22d73061ae 100644 --- a/tests/integration/test_materialized_mysql_database/test.py +++ b/tests/integration/test_materialized_mysql_database/test.py @@ -516,3 +516,10 @@ def test_savepoint_query( ): materialize_with_ddl.savepoint(clickhouse_node, started_mysql_8_0, "mysql80") materialize_with_ddl.savepoint(clickhouse_node, started_mysql_5_7, "mysql57") + + +def test_materialized_database_mysql_drop_ddl( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.dropddl(clickhouse_node, started_mysql_8_0, "mysql80") + materialize_with_ddl.dropddl(clickhouse_node, started_mysql_5_7, "mysql57") diff --git a/tests/integration/test_merge_tree_empty_parts/test.py b/tests/integration/test_merge_tree_empty_parts/test.py index 57bf49e6803..0f611408a67 100644 --- a/tests/integration/test_merge_tree_empty_parts/test.py +++ b/tests/integration/test_merge_tree_empty_parts/test.py @@ -24,8 +24,10 @@ def started_cluster(): def test_empty_parts_alter_delete(started_cluster): node1.query( - "CREATE TABLE empty_parts_delete (d Date, key UInt64, value String) \ - ENGINE = ReplicatedMergeTree('/clickhouse/tables/empty_parts_delete', 'r1') PARTITION BY toYYYYMM(d) ORDER BY key" + "CREATE TABLE empty_parts_delete (d Date, key UInt64, value String) " + "ENGINE = ReplicatedMergeTree('/clickhouse/tables/empty_parts_delete', 'r1') " + "PARTITION BY toYYYYMM(d) ORDER BY key " + "SETTINGS old_parts_lifetime = 1" ) node1.query("INSERT INTO empty_parts_delete VALUES (toDate('2020-10-10'), 1, 'a')") @@ -43,8 +45,10 @@ def test_empty_parts_alter_delete(started_cluster): def test_empty_parts_summing(started_cluster): node1.query( - "CREATE TABLE empty_parts_summing (d Date, key UInt64, value Int64) \ - ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/empty_parts_summing', 'r1') PARTITION BY toYYYYMM(d) ORDER BY key" + "CREATE TABLE empty_parts_summing (d Date, key UInt64, value Int64) " + "ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/empty_parts_summing', 'r1') " + "PARTITION BY toYYYYMM(d) ORDER BY key " + "SETTINGS old_parts_lifetime = 1" ) node1.query("INSERT INTO empty_parts_summing VALUES (toDate('2020-10-10'), 1, 1)") diff --git a/tests/integration/test_merge_tree_hdfs/test.py b/tests/integration/test_merge_tree_hdfs/test.py index 132e1027586..9edb71ec15a 100644 --- a/tests/integration/test_merge_tree_hdfs/test.py +++ b/tests/integration/test_merge_tree_hdfs/test.py @@ -5,6 +5,8 @@ import os import pytest from helpers.cluster import ClickHouseCluster from helpers.utility import generate_values +from helpers.wait_for_helpers import wait_for_delete_inactive_parts +from helpers.wait_for_helpers import wait_for_delete_empty_parts from pyhdfs import HdfsClient @@ -209,6 +211,8 @@ def test_attach_detach_partition(cluster): node.query("ALTER TABLE hdfs_test DETACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(4096)" + wait_for_delete_inactive_parts(node, "hdfs_test") + wait_for_delete_empty_parts(node, "hdfs_test") hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 @@ -221,6 +225,8 @@ def test_attach_detach_partition(cluster): node.query("ALTER TABLE hdfs_test DROP PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(4096)" + wait_for_delete_inactive_parts(node, "hdfs_test") + wait_for_delete_empty_parts(node, "hdfs_test") hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE @@ -231,6 +237,8 @@ def test_attach_detach_partition(cluster): settings={"allow_drop_detached": 1}, ) assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(0)" + wait_for_delete_inactive_parts(node, "hdfs_test") + wait_for_delete_empty_parts(node, "hdfs_test") hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD @@ -297,6 +305,8 @@ def test_table_manipulations(cluster): node.query("TRUNCATE TABLE hdfs_test") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(0)" + wait_for_delete_inactive_parts(node, "hdfs_test") + wait_for_delete_empty_parts(node, "hdfs_test") hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index b2e93db2606..002bc8ec9d7 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -5,6 +5,9 @@ import os import pytest from helpers.cluster import ClickHouseCluster from helpers.utility import generate_values, replace_config, SafeThread +from helpers.wait_for_helpers import wait_for_delete_inactive_parts +from helpers.wait_for_helpers import wait_for_delete_empty_parts + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -320,6 +323,8 @@ def test_attach_detach_partition(cluster, node_name): ) node.query("ALTER TABLE s3_test DETACH PARTITION '2020-01-03'") + wait_for_delete_inactive_parts(node, "s3_test") + wait_for_delete_empty_parts(node, "s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" assert ( len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) @@ -334,13 +339,22 @@ def test_attach_detach_partition(cluster, node_name): ) node.query("ALTER TABLE s3_test DROP PARTITION '2020-01-03'") + wait_for_delete_inactive_parts(node, "s3_test") + wait_for_delete_empty_parts(node, "s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" assert ( len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 1 ) node.query("ALTER TABLE s3_test DETACH PARTITION '2020-01-04'") + wait_for_delete_inactive_parts(node, "s3_test") + wait_for_delete_empty_parts(node, "s3_test") + assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 1 + ) node.query( "ALTER TABLE s3_test DROP DETACHED PARTITION '2020-01-04'", settings={"allow_drop_detached": 1}, @@ -348,7 +362,7 @@ def test_attach_detach_partition(cluster, node_name): assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" assert ( len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 0 ) @@ -417,6 +431,8 @@ def test_table_manipulations(cluster, node_name): ) node.query("TRUNCATE TABLE s3_test") + wait_for_delete_inactive_parts(node, "s3_test") + wait_for_delete_empty_parts(node, "s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" assert ( len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) @@ -530,6 +546,8 @@ def test_freeze_unfreeze(cluster, node_name): node.query("ALTER TABLE s3_test FREEZE WITH NAME 'backup2'") node.query("TRUNCATE TABLE s3_test") + wait_for_delete_inactive_parts(node, "s3_test") + wait_for_delete_empty_parts(node, "s3_test") assert ( len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 @@ -568,6 +586,8 @@ def test_freeze_system_unfreeze(cluster, node_name): node.query("ALTER TABLE s3_test_removed FREEZE WITH NAME 'backup3'") node.query("TRUNCATE TABLE s3_test") + wait_for_delete_inactive_parts(node, "s3_test") + wait_for_delete_empty_parts(node, "s3_test") node.query("DROP TABLE s3_test_removed NO DELAY") assert ( len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) @@ -763,7 +783,7 @@ def test_cache_setting_compatibility(cluster, node_name): node.query("DROP TABLE IF EXISTS s3_test NO DELAY") node.query( - "CREATE TABLE s3_test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_r';" + "CREATE TABLE s3_test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_r', compress_marks=false, compress_primary_key=false;" ) node.query( "INSERT INTO s3_test SELECT * FROM generateRandom('key UInt32, value String') LIMIT 500" diff --git a/tests/integration/test_merge_tree_s3_restore/test.py b/tests/integration/test_merge_tree_s3_restore/test.py index 0652c31951d..d29bb1e34ac 100644 --- a/tests/integration/test_merge_tree_s3_restore/test.py +++ b/tests/integration/test_merge_tree_s3_restore/test.py @@ -6,6 +6,8 @@ import time import pytest from helpers.cluster import ClickHouseCluster +from helpers.wait_for_helpers import wait_for_delete_empty_parts +from helpers.wait_for_helpers import wait_for_delete_inactive_parts SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -103,8 +105,8 @@ def create_table( ORDER BY (dt, id) SETTINGS storage_policy='s3', - old_parts_lifetime=600, - index_granularity=512 + index_granularity=512, + old_parts_lifetime=1 """.format( create="ATTACH" if attach else "CREATE", table_name=table_name, @@ -142,6 +144,7 @@ def create_restore_file(node, revision=None, bucket=None, path=None, detached=No node.exec_in_container( ["bash", "-c", "mkdir -p /var/lib/clickhouse/disks/s3/"], user="root" ) + node.exec_in_container( ["bash", "-c", "touch /var/lib/clickhouse/disks/s3/restore"], user="root" ) @@ -270,6 +273,7 @@ def test_restore_another_bucket_path(cluster, db_atomic): # To ensure parts have merged node.query("OPTIMIZE TABLE s3.test") + wait_for_delete_inactive_parts(node, "s3.test", retry_count=120) assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format( 4096 * 4 @@ -336,6 +340,9 @@ def test_restore_different_revisions(cluster, db_atomic): # To ensure parts have merged node.query("OPTIMIZE TABLE s3.test") + wait_for_delete_inactive_parts(node, "s3.test", retry_count=120) + + assert node.query("SELECT count(*) from system.parts where table = 'test'") == "3\n" node.query("ALTER TABLE s3.test FREEZE") revision3 = get_revision_counter(node, 3) @@ -344,7 +351,7 @@ def test_restore_different_revisions(cluster, db_atomic): 4096 * 4 ) assert node.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) - assert node.query("SELECT count(*) from system.parts where table = 'test'") == "5\n" + assert node.query("SELECT count(*) from system.parts where table = 'test'") == "3\n" node_another_bucket = cluster.instances["node_another_bucket"] @@ -403,7 +410,7 @@ def test_restore_different_revisions(cluster, db_atomic): node_another_bucket.query( "SELECT count(*) from system.parts where table = 'test'" ) - == "5\n" + == "3\n" ) @@ -593,6 +600,8 @@ def test_restore_to_detached(cluster, replicated, db_atomic): # Detach some partition. node.query("ALTER TABLE s3.test DETACH PARTITION '2020-01-07'") + wait_for_delete_empty_parts(node, "s3.test", retry_count=120) + wait_for_delete_inactive_parts(node, "s3.test", retry_count=120) node.query("ALTER TABLE s3.test FREEZE") revision = get_revision_counter(node, 1) @@ -623,10 +632,10 @@ def test_restore_to_detached(cluster, replicated, db_atomic): node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-04'") node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-05'") node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-06'") - assert node_another_bucket.query( "SELECT count(*) FROM s3.test FORMAT Values" ) == "({})".format(4096 * 4) + assert node_another_bucket.query( "SELECT sum(id) FROM s3.test FORMAT Values" ) == "({})".format(0) diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index d7117e2546a..9b7bad2b256 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -1244,10 +1244,16 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): def alter_drop(num): for i in range(num): partition = random.choice([201903, 201904]) - drach = random.choice(["drop", "detach"]) - node1.query( - "ALTER TABLE {} {} PARTITION {}".format(name, drach, partition) - ) + op = random.choice(["drop", "detach"]) + try: + node1.query( + "ALTER TABLE {} {} PARTITION {}".format(name, op, partition) + ) + except QueryRuntimeException as e: + if "Code: 650" in e.stderr: + pass + else: + raise e insert(100) p = Pool(15) @@ -1655,7 +1661,7 @@ def test_freeze(start_cluster): ) ENGINE = MergeTree ORDER BY tuple() PARTITION BY toYYYYMM(d) - SETTINGS storage_policy='small_jbod_with_external' + SETTINGS storage_policy='small_jbod_with_external', compress_marks=false, compress_primary_key=false """ ) diff --git a/tests/integration/test_named_collections/__init__.py b/tests/integration/test_named_collections/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_named_collections/configs/config.d/named_collections.xml b/tests/integration/test_named_collections/configs/config.d/named_collections.xml new file mode 100644 index 00000000000..d24fb303b37 --- /dev/null +++ b/tests/integration/test_named_collections/configs/config.d/named_collections.xml @@ -0,0 +1,7 @@ + + + + value1 + + + diff --git a/tests/integration/test_named_collections/configs/users.d/users.xml b/tests/integration/test_named_collections/configs/users.d/users.xml new file mode 100644 index 00000000000..ee38baa3df9 --- /dev/null +++ b/tests/integration/test_named_collections/configs/users.d/users.xml @@ -0,0 +1,13 @@ + + + + + + ::/0 + + default + default + 1 + + + diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py new file mode 100644 index 00000000000..ce5c8aaa62e --- /dev/null +++ b/tests/integration/test_named_collections/test.py @@ -0,0 +1,200 @@ +import logging +import pytest +import os +import time +from helpers.cluster import ClickHouseCluster + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +NAMED_COLLECTIONS_CONFIG = os.path.join( + SCRIPT_DIR, "./configs/config.d/named_collections.xml" +) + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "node", + main_configs=[ + "configs/config.d/named_collections.xml", + ], + user_configs=[ + "configs/users.d/users.xml", + ], + stay_alive=True, + ) + + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + yield cluster + finally: + cluster.shutdown() + + +def replace_config(node, old, new): + node.replace_in_config( + "/etc/clickhouse-server/config.d/named_collections.xml", + old, + new, + ) + + +def test_config_reload(cluster): + node = cluster.instances["node"] + assert ( + "collection1" == node.query("select name from system.named_collections").strip() + ) + assert ( + "['key1']" + == node.query( + "select mapKeys(collection) from system.named_collections where name = 'collection1'" + ).strip() + ) + assert ( + "value1" + == node.query( + "select collection['key1'] from system.named_collections where name = 'collection1'" + ).strip() + ) + + replace_config(node, "value1", "value2") + node.query("SYSTEM RELOAD CONFIG") + + assert ( + "['key1']" + == node.query( + "select mapKeys(collection) from system.named_collections where name = 'collection1'" + ).strip() + ) + assert ( + "value2" + == node.query( + "select collection['key1'] from system.named_collections where name = 'collection1'" + ).strip() + ) + + +def test_sql_commands(cluster): + node = cluster.instances["node"] + assert "1" == node.query("select count() from system.named_collections").strip() + + node.query("CREATE NAMED COLLECTION collection2 AS key1=1, key2='value2'") + + def check_created(): + assert ( + "collection1\ncollection2" + == node.query("select name from system.named_collections").strip() + ) + + assert ( + "['key1','key2']" + == node.query( + "select mapKeys(collection) from system.named_collections where name = 'collection2'" + ).strip() + ) + + assert ( + "1" + == node.query( + "select collection['key1'] from system.named_collections where name = 'collection2'" + ).strip() + ) + + assert ( + "value2" + == node.query( + "select collection['key2'] from system.named_collections where name = 'collection2'" + ).strip() + ) + + check_created() + node.restart_clickhouse() + check_created() + + node.query("ALTER NAMED COLLECTION collection2 SET key1=4, key3='value3'") + + def check_altered(): + assert ( + "['key1','key2','key3']" + == node.query( + "select mapKeys(collection) from system.named_collections where name = 'collection2'" + ).strip() + ) + + assert ( + "4" + == node.query( + "select collection['key1'] from system.named_collections where name = 'collection2'" + ).strip() + ) + + assert ( + "value3" + == node.query( + "select collection['key3'] from system.named_collections where name = 'collection2'" + ).strip() + ) + + check_altered() + node.restart_clickhouse() + check_altered() + + node.query("ALTER NAMED COLLECTION collection2 DELETE key2") + + def check_deleted(): + assert ( + "['key1','key3']" + == node.query( + "select mapKeys(collection) from system.named_collections where name = 'collection2'" + ).strip() + ) + + check_deleted() + node.restart_clickhouse() + check_deleted() + + node.query( + "ALTER NAMED COLLECTION collection2 SET key3=3, key4='value4' DELETE key1" + ) + + def check_altered_and_deleted(): + assert ( + "['key3','key4']" + == node.query( + "select mapKeys(collection) from system.named_collections where name = 'collection2'" + ).strip() + ) + + assert ( + "3" + == node.query( + "select collection['key3'] from system.named_collections where name = 'collection2'" + ).strip() + ) + + assert ( + "value4" + == node.query( + "select collection['key4'] from system.named_collections where name = 'collection2'" + ).strip() + ) + + check_altered_and_deleted() + node.restart_clickhouse() + check_altered_and_deleted() + + node.query("DROP NAMED COLLECTION collection2") + + def check_dropped(): + assert "1" == node.query("select count() from system.named_collections").strip() + assert ( + "collection1" + == node.query("select name from system.named_collections").strip() + ) + + check_dropped() + node.restart_clickhouse() + check_dropped() diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index c53bc5a9d0d..6bd224851e7 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -3,6 +3,8 @@ import logging from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV from helpers.test_tools import assert_eq_with_retry +from helpers.wait_for_helpers import wait_for_delete_inactive_parts +from helpers.wait_for_helpers import wait_for_delete_empty_parts cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( @@ -36,7 +38,7 @@ def partition_table_simple(started_cluster): q( "CREATE TABLE test.partition_simple (date MATERIALIZED toDate(0), x UInt64, sample_key MATERIALIZED intHash64(x)) " "ENGINE=MergeTree PARTITION BY date SAMPLE BY sample_key ORDER BY (date,x,sample_key) " - "SETTINGS index_granularity=8192, index_granularity_bytes=0" + "SETTINGS index_granularity=8192, index_granularity_bytes=0, compress_marks=false, compress_primary_key=false" ) q("INSERT INTO test.partition_simple ( x ) VALUES ( now() )") q("INSERT INTO test.partition_simple ( x ) VALUES ( now()+1 )") @@ -115,7 +117,7 @@ def partition_table_complex(started_cluster): q("DROP TABLE IF EXISTS test.partition_complex") q( "CREATE TABLE test.partition_complex (p Date, k Int8, v1 Int8 MATERIALIZED k + 1) " - "ENGINE = MergeTree PARTITION BY p ORDER BY k SETTINGS index_granularity=1, index_granularity_bytes=0" + "ENGINE = MergeTree PARTITION BY p ORDER BY k SETTINGS index_granularity=1, index_granularity_bytes=0, compress_marks=false, compress_primary_key=false" ) q("INSERT INTO test.partition_complex (p, k) VALUES(toDate(31), 1)") q("INSERT INTO test.partition_complex (p, k) VALUES(toDate(1), 2)") @@ -153,7 +155,7 @@ def test_partition_complex(partition_table_complex): def cannot_attach_active_part_table(started_cluster): q("DROP TABLE IF EXISTS test.attach_active") q( - "CREATE TABLE test.attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n" + "CREATE TABLE test.attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n SETTINGS compress_marks=false, compress_primary_key=false" ) q("INSERT INTO test.attach_active SELECT number FROM system.numbers LIMIT 16") @@ -181,7 +183,7 @@ def attach_check_all_parts_table(started_cluster): q("SYSTEM STOP MERGES") q("DROP TABLE IF EXISTS test.attach_partition") q( - "CREATE TABLE test.attach_partition (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n" + "CREATE TABLE test.attach_partition (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n SETTINGS compress_marks=false, compress_primary_key=false" ) q( "INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8" @@ -199,6 +201,9 @@ def attach_check_all_parts_table(started_cluster): def test_attach_check_all_parts(attach_check_all_parts_table): q("ALTER TABLE test.attach_partition DETACH PARTITION 0") + wait_for_delete_inactive_parts(instance, "test.attach_partition") + wait_for_delete_empty_parts(instance, "test.attach_partition") + path_to_detached = path_to_data + "data/test/attach_partition/detached/" instance.exec_in_container(["mkdir", "{}".format(path_to_detached + "0_5_5_0")]) instance.exec_in_container( @@ -226,7 +231,8 @@ def test_attach_check_all_parts(attach_check_all_parts_table): ) parts = q( - "SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name" + "SElECT name FROM system.parts " + "WHERE table='attach_partition' AND database='test' AND active ORDER BY name" ) assert TSV(parts) == TSV("1_2_2_0\n1_4_4_0") detached = q( @@ -259,7 +265,7 @@ def drop_detached_parts_table(started_cluster): q("SYSTEM STOP MERGES") q("DROP TABLE IF EXISTS test.drop_detached") q( - "CREATE TABLE test.drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n" + "CREATE TABLE test.drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n SETTINGS compress_marks=false, compress_primary_key=false" ) q( "INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8" @@ -329,9 +335,15 @@ def test_drop_detached_parts(drop_detached_parts_table): def test_system_detached_parts(drop_detached_parts_table): - q("create table sdp_0 (n int, x int) engine=MergeTree order by n") - q("create table sdp_1 (n int, x int) engine=MergeTree order by n partition by x") - q("create table sdp_2 (n int, x String) engine=MergeTree order by n partition by x") + q( + "create table sdp_0 (n int, x int) engine=MergeTree order by n SETTINGS compress_marks=false, compress_primary_key=false" + ) + q( + "create table sdp_1 (n int, x int) engine=MergeTree order by n partition by x SETTINGS compress_marks=false, compress_primary_key=false" + ) + q( + "create table sdp_2 (n int, x String) engine=MergeTree order by n partition by x SETTINGS compress_marks=false, compress_primary_key=false" + ) q( "create table sdp_3 (n int, x Enum('broken' = 0, 'all' = 1)) engine=MergeTree order by n partition by x" ) @@ -449,15 +461,20 @@ def test_system_detached_parts(drop_detached_parts_table): def test_detached_part_dir_exists(started_cluster): - q("create table detached_part_dir_exists (n int) engine=MergeTree order by n") + q( + "create table detached_part_dir_exists (n int) engine=MergeTree order by n SETTINGS compress_marks=false, compress_primary_key=false" + ) q("insert into detached_part_dir_exists select 1") # will create all_1_1_0 q( "alter table detached_part_dir_exists detach partition id 'all'" - ) # will move all_1_1_0 to detached/all_1_1_0 + ) # will move all_1_1_0 to detached/all_1_1_0 and create all_1_1_1 + + wait_for_delete_empty_parts(instance, "detached_part_dir_exists") + q("detach table detached_part_dir_exists") q("attach table detached_part_dir_exists") - q("insert into detached_part_dir_exists select 1") # will create all_1_1_0 q("insert into detached_part_dir_exists select 1") # will create all_2_2_0 + q("insert into detached_part_dir_exists select 1") # will create all_3_3_0 instance.exec_in_container( [ "bash", @@ -488,7 +505,7 @@ def test_detached_part_dir_exists(started_cluster): def test_make_clone_in_detached(started_cluster): q( - "create table clone_in_detached (n int, m String) engine=ReplicatedMergeTree('/clone_in_detached', '1') order by n" + "create table clone_in_detached (n int, m String) engine=ReplicatedMergeTree('/clone_in_detached', '1') order by n SETTINGS compress_marks=false, compress_primary_key=false" ) path = path_to_data + "data/default/clone_in_detached/" diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index 32b5e531fa8..361b4855747 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -728,7 +728,7 @@ def test_polymorphic_parts_index(start_cluster): """ CREATE TABLE test_index.index_compact(a UInt32, s String) ENGINE = MergeTree ORDER BY a - SETTINGS min_rows_for_wide_part = 1000, index_granularity = 128, merge_max_block_size = 100""" + SETTINGS min_rows_for_wide_part = 1000, index_granularity = 128, merge_max_block_size = 100, compress_marks=false, compress_primary_key=false""" ) node1.query( diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index de5433d5beb..1e6a39ee1bd 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -592,60 +592,64 @@ def test_alters_from_different_replicas(started_cluster): def create_some_tables(db): settings = {"distributed_ddl_task_timeout": 0} - main_node.query( - "CREATE TABLE {}.t1 (n int) ENGINE=Memory".format(db), settings=settings - ) + main_node.query(f"CREATE TABLE {db}.t1 (n int) ENGINE=Memory", settings=settings) dummy_node.query( - "CREATE TABLE {}.t2 (s String) ENGINE=Memory".format(db), settings=settings + f"CREATE TABLE {db}.t2 (s String) ENGINE=Memory", settings=settings ) main_node.query( - "CREATE TABLE {}.mt1 (n int) ENGINE=MergeTree order by n".format(db), + f"CREATE TABLE {db}.mt1 (n int) ENGINE=MergeTree order by n", settings=settings, ) dummy_node.query( - "CREATE TABLE {}.mt2 (n int) ENGINE=MergeTree order by n".format(db), + f"CREATE TABLE {db}.mt2 (n int) ENGINE=MergeTree order by n", settings=settings, ) main_node.query( - "CREATE TABLE {}.rmt1 (n int) ENGINE=ReplicatedMergeTree order by n".format(db), + f"CREATE TABLE {db}.rmt1 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings, ) dummy_node.query( - "CREATE TABLE {}.rmt2 (n int) ENGINE=ReplicatedMergeTree order by n".format(db), + f"CREATE TABLE {db}.rmt2 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings, ) main_node.query( - "CREATE TABLE {}.rmt3 (n int) ENGINE=ReplicatedMergeTree order by n".format(db), + f"CREATE TABLE {db}.rmt3 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings, ) dummy_node.query( - "CREATE TABLE {}.rmt5 (n int) ENGINE=ReplicatedMergeTree order by n".format(db), + f"CREATE TABLE {db}.rmt5 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings, ) main_node.query( - "CREATE MATERIALIZED VIEW {}.mv1 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt1".format( - db - ), + f"CREATE MATERIALIZED VIEW {db}.mv1 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt1", settings=settings, ) dummy_node.query( - "CREATE MATERIALIZED VIEW {}.mv2 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt2".format( - db - ), + f"CREATE MATERIALIZED VIEW {db}.mv2 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt2", settings=settings, ) main_node.query( - "CREATE DICTIONARY {}.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " + f"CREATE DICTIONARY {db}.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " - "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())".format(db) + "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" ) dummy_node.query( - "CREATE DICTIONARY {}.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " + f"CREATE DICTIONARY {db}.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB 'recover')) " - "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())".format(db) + "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" ) +# These tables are used to check that DatabaseReplicated correctly renames all the tables in case when it restores from the lost state +def create_table_for_exchanges(db): + settings = {"distributed_ddl_task_timeout": 0} + for table in ["a1", "a2", "a3", "a4", "a5", "a6"]: + main_node.query( + f"CREATE TABLE {db}.{table} (s String) ENGINE=ReplicatedMergeTree order by s", + settings=settings, + ) + + def test_recover_staled_replica(started_cluster): main_node.query( "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica1');" @@ -659,13 +663,20 @@ def test_recover_staled_replica(started_cluster): settings = {"distributed_ddl_task_timeout": 0} create_some_tables("recover") + create_table_for_exchanges("recover") for table in ["t1", "t2", "mt1", "mt2", "rmt1", "rmt2", "rmt3", "rmt5"]: - main_node.query("INSERT INTO recover.{} VALUES (42)".format(table)) + main_node.query(f"INSERT INTO recover.{table} VALUES (42)") for table in ["t1", "t2", "mt1", "mt2"]: - dummy_node.query("INSERT INTO recover.{} VALUES (42)".format(table)) + dummy_node.query(f"INSERT INTO recover.{table} VALUES (42)") + + for i, table in enumerate(["a1", "a2", "a3", "a4", "a5", "a6"]): + main_node.query(f"INSERT INTO recover.{table} VALUES ('{str(i + 1) * 10}')") + for table in ["rmt1", "rmt2", "rmt3", "rmt5"]: - main_node.query("SYSTEM SYNC REPLICA recover.{}".format(table)) + main_node.query(f"SYSTEM SYNC REPLICA recover.{table}") + for table in ["a1", "a2", "a3", "a4", "a5", "a6"]: + main_node.query(f"SYSTEM SYNC REPLICA recover.{table}") with PartitionManager() as pm: pm.drop_instance_zk_connections(dummy_node) @@ -699,19 +710,15 @@ def test_recover_staled_replica(started_cluster): ).strip() ) main_node.query_with_retry( - "ALTER TABLE recover.`{}` MODIFY COLUMN n int DEFAULT 42".format( - inner_table - ), + f"ALTER TABLE recover.`{inner_table}` MODIFY COLUMN n int DEFAULT 42", settings=settings, ) main_node.query_with_retry( - "ALTER TABLE recover.mv1 MODIFY QUERY SELECT m FROM recover.rmt1".format( - inner_table - ), + "ALTER TABLE recover.mv1 MODIFY QUERY SELECT m FROM recover.rmt1", settings=settings, ) main_node.query_with_retry( - "RENAME TABLE recover.mv2 TO recover.mv3".format(inner_table), + "RENAME TABLE recover.mv2 TO recover.mv3", settings=settings, ) @@ -727,11 +734,18 @@ def test_recover_staled_replica(started_cluster): "CREATE TABLE recover.tmp AS recover.m1", settings=settings ) + main_node.query("EXCHANGE TABLES recover.a1 AND recover.a2", settings=settings) + main_node.query("EXCHANGE TABLES recover.a3 AND recover.a4", settings=settings) + main_node.query("EXCHANGE TABLES recover.a5 AND recover.a4", settings=settings) + main_node.query("EXCHANGE TABLES recover.a6 AND recover.a3", settings=settings) + main_node.query("RENAME TABLE recover.a6 TO recover.a7", settings=settings) + main_node.query("RENAME TABLE recover.a1 TO recover.a8", settings=settings) + assert ( main_node.query( "SELECT name FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' ORDER BY name" ) - == "d1\nd2\nm1\nmt1\nmt2\nmv1\nmv3\nrmt1\nrmt2\nrmt4\nt2\ntmp\n" + == "a2\na3\na4\na5\na7\na8\nd1\nd2\nm1\nmt1\nmt2\nmv1\nmv3\nrmt1\nrmt2\nrmt4\nt2\ntmp\n" ) query = ( "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' " @@ -752,6 +766,12 @@ def test_recover_staled_replica(started_cluster): == "2\n" ) + # Check that Database Replicated renamed all the tables correctly + for i, table in enumerate(["a2", "a8", "a5", "a7", "a4", "a3"]): + assert ( + dummy_node.query(f"SELECT * FROM recover.{table}") == f"{str(i + 1) * 10}\n" + ) + for table in [ "m1", "t2", @@ -765,11 +785,11 @@ def test_recover_staled_replica(started_cluster): "mv1", "mv3", ]: - assert main_node.query("SELECT (*,).1 FROM recover.{}".format(table)) == "42\n" + assert main_node.query(f"SELECT (*,).1 FROM recover.{table}") == "42\n" for table in ["t2", "rmt1", "rmt2", "rmt4", "d1", "d2", "mt2", "mv1", "mv3"]: - assert dummy_node.query("SELECT (*,).1 FROM recover.{}".format(table)) == "42\n" + assert dummy_node.query(f"SELECT (*,).1 FROM recover.{table}") == "42\n" for table in ["m1", "mt1"]: - assert dummy_node.query("SELECT count() FROM recover.{}".format(table)) == "0\n" + assert dummy_node.query(f"SELECT count() FROM recover.{table}") == "0\n" global test_recover_staled_replica_run assert ( dummy_node.query( @@ -784,20 +804,22 @@ def test_recover_staled_replica(started_cluster): == f"{test_recover_staled_replica_run}\n" ) test_recover_staled_replica_run += 1 + + print(dummy_node.query("SHOW DATABASES")) + print(dummy_node.query("SHOW TABLES FROM recover_broken_tables")) + print(dummy_node.query("SHOW TABLES FROM recover_broken_replicated_tables")) + table = dummy_node.query( - "SHOW TABLES FROM recover_broken_tables LIKE 'mt1_29_%' LIMIT 1" + "SHOW TABLES FROM recover_broken_tables LIKE 'mt1_41_%' LIMIT 1" ).strip() assert ( - dummy_node.query("SELECT (*,).1 FROM recover_broken_tables.{}".format(table)) - == "42\n" + dummy_node.query(f"SELECT (*,).1 FROM recover_broken_tables.{table}") == "42\n" ) table = dummy_node.query( - "SHOW TABLES FROM recover_broken_replicated_tables LIKE 'rmt5_29_%' LIMIT 1" + "SHOW TABLES FROM recover_broken_replicated_tables LIKE 'rmt5_41_%' LIMIT 1" ).strip() assert ( - dummy_node.query( - "SELECT (*,).1 FROM recover_broken_replicated_tables.{}".format(table) - ) + dummy_node.query(f"SELECT (*,).1 FROM recover_broken_replicated_tables.{table}") == "42\n" ) diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 3358315cca7..335a0db53c0 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -550,7 +550,7 @@ def test_function_current_profiles(): user="robin", params={"session_id": session_id}, ) - == "['P1','P2']\t['P1','P2']\t['default','P3','P4','P5','P1','P2']\n" + == "['P1','P2']\t['default','P3','P5','P1','P2']\t['default','P3','P4','P5','P1','P2']\n" ) instance.http_query( diff --git a/tests/integration/test_transactions/test.py b/tests/integration/test_transactions/test.py index daa4c287982..7902d168707 100644 --- a/tests/integration/test_transactions/test.py +++ b/tests/integration/test_transactions/test.py @@ -104,6 +104,8 @@ def test_rollback_unfinished_on_restart1(start_cluster): "0_4_4_0_7\t0\ttid3\tcsn18446744073709551615_\ttid0\tcsn0_\n" "0_8_8_0\t0\ttid5\tcsn18446744073709551615_\ttid0\tcsn0_\n" "1_1_1_0\t0\ttid0\tcsn1_\ttid1\tcsn_1\n" + "1_1_1_1\t1\ttid1\tcsn_1\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" + "1_1_1_1_7\t0\ttid3\tcsn18446744073709551615_\ttid0\tcsn0_\n" "1_3_3_0\t1\ttid2\tcsn_2\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" "1_3_3_0_7\t0\ttid3\tcsn18446744073709551615_\ttid0\tcsn0_\n" "1_5_5_0\t1\ttid6\tcsn_6\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" @@ -190,5 +192,6 @@ def test_rollback_unfinished_on_restart2(start_cluster): "0_4_4_0\t1\ttid2\tcsn_2\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" "0_5_5_0\t0\ttid5\tcsn18446744073709551615_\ttid0\tcsn0_\n" "1_1_1_0\t0\ttid0\tcsn1_\ttid1\tcsn_1\n" + "1_1_1_1\t1\ttid1\tcsn_1\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" "1_3_3_0\t1\ttid2\tcsn_2\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" ) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index cacd9ef0c78..aa4a09f1269 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -4,6 +4,8 @@ import helpers.client as client import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV, exec_query_with_retry +from helpers.wait_for_helpers import wait_for_delete_inactive_parts +from helpers.wait_for_helpers import wait_for_delete_empty_parts cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", with_zookeeper=True) @@ -420,7 +422,8 @@ def test_ttl_empty_parts(started_cluster): ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_empty_parts', '{replica}') ORDER BY id SETTINGS max_bytes_to_merge_at_min_space_in_pool = 1, max_bytes_to_merge_at_max_space_in_pool = 1, - cleanup_delay_period = 1, cleanup_delay_period_random_add = 0 + cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, old_parts_lifetime = 1 + """.format( replica=node.name ) @@ -445,7 +448,10 @@ def test_ttl_empty_parts(started_cluster): assert node1.query("SELECT count() FROM test_ttl_empty_parts") == "3000\n" - time.sleep(3) # Wait for cleanup thread + # Wait for cleanup thread + wait_for_delete_inactive_parts(node1, "test_ttl_empty_parts") + wait_for_delete_empty_parts(node1, "test_ttl_empty_parts") + assert ( node1.query( "SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name" diff --git a/tests/performance/async_remote_read.xml b/tests/performance/async_remote_read.xml index 4ea159f9a97..bc28b1c6e50 100644 --- a/tests/performance/async_remote_read.xml +++ b/tests/performance/async_remote_read.xml @@ -11,4 +11,8 @@ ) SETTINGS max_threads = 2, max_distributed_connections = 2 + + + select sum(length(URL)) from hits_10m_single settings max_threads=2, max_streams_to_max_threads_ratio=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1 + select sum(length(URL)) from hits_10m_single settings max_threads=2, max_streams_for_merge_tree_reading=32, allow_asynchronous_read_from_io_pool_for_merge_tree=1 diff --git a/tests/performance/general_purpose_hashes.xml b/tests/performance/general_purpose_hashes.xml index f34554360cf..ba4e8f93859 100644 --- a/tests/performance/general_purpose_hashes.xml +++ b/tests/performance/general_purpose_hashes.xml @@ -15,6 +15,7 @@ hiveHash xxHash32 xxHash64 + xxh3 CRC32 diff --git a/tests/performance/grace_hash_join.xml b/tests/performance/grace_hash_join.xml new file mode 100644 index 00000000000..8b28f9d7414 --- /dev/null +++ b/tests/performance/grace_hash_join.xml @@ -0,0 +1,21 @@ + + + 16 + 10G + + + + + settings + + join_algorithm='hash' + join_algorithm='parallel_hash' + join_algorithm='partial_merge', max_bytes_in_join='1G' + join_algorithm='grace_hash', max_bytes_in_join='100M' + + + + + SELECT sum(n) FROM (SELECT number * 2 AS n FROM numbers_mt(10000000)) AS lhs JOIN (SELECT number * 3 AS n FROM numbers_mt(10000000)) AS rhs USING (n) SETTINGS {settings} FORMAT Null + SELECT sum(n) FROM (SELECT intHash64(number * 2) AS n FROM numbers_mt(10000000)) AS lhs JOIN (SELECT intHash64(number * 3) AS n FROM numbers_mt(10000000)) AS rhs USING (n) SETTINGS {settings} FORMAT Null + diff --git a/tests/performance/memory_bound_merging.xml b/tests/performance/memory_bound_merging.xml new file mode 100644 index 00000000000..3b13400151c --- /dev/null +++ b/tests/performance/memory_bound_merging.xml @@ -0,0 +1,17 @@ + + + 1 + 1 + + + create table t_mbm(a UInt64) engine=MergeTree order by a + + insert into t_mbm select * from numbers_mt(5e6) + optimize table t_mbm final + + select avg(a) from remote('127.0.0.{{1,2}}', default, t_mbm) group by a format Null + + select * from remote('127.0.0.{{1,2}}', default, t_mbm) group by a format Null settings allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, use_hedged_requests = 0 + + drop table t_mbm + diff --git a/tests/performance/query_interpretation_join.xml b/tests/performance/query_interpretation_join.xml new file mode 100644 index 00000000000..5bbb0baf842 --- /dev/null +++ b/tests/performance/query_interpretation_join.xml @@ -0,0 +1,393 @@ + + + CREATE TABLE IF NOT EXISTS interpret_table_01 + ( + `idColumnU64` UInt64, + `dateColumn` DateTime, + `aggCount` AggregateFunction(count), + `aggArgMaxFloat32_1` AggregateFunction(argMax, Float32, DateTime), + `aggArgMaxString` AggregateFunction(argMax, String, DateTime), + `aggArgMaxFloat32_2` AggregateFunction(argMax, Float32, DateTime), + `nDateTime_02_date` SimpleAggregateFunction(max, DateTime), + `nDateTime_02_date_292929292` SimpleAggregateFunction(max, DateTime), + `agg_topk_uint32` AggregateFunction(topKWeighted(2), UInt32, UInt32), + `agg_argmax_string_datetime_01` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_u8_01` AggregateFunction(argMax, UInt8, DateTime), + `agg_argmax_string_datetime_02` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_datetime_03` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_datetime_04` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_datetime_05` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_datetime_06` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_datetime_07` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_u8_02` AggregateFunction(argMax, UInt8, DateTime), + `agg_argmax_string_u8_03` AggregateFunction(argMax, UInt8, DateTime), + `agg_argmax_string_u8_04` AggregateFunction(argMax, UInt8, DateTime), + `agg_count_01` AggregateFunction(count), + `agg_count_02` AggregateFunction(count), + `agg_count_03` AggregateFunction(count), + `agg_count_04` AggregateFunction(count), + `agg_count_05` AggregateFunction(count), + `agg_count_06` AggregateFunction(count), + `agg_count_07` AggregateFunction(count), + `agg_count_08` AggregateFunction(count), + `agg_count_09` AggregateFunction(count), + `agg_count_10` AggregateFunction(count), + `agg_count_11` AggregateFunction(count), + `agg_count_12` AggregateFunction(count), + `agg_count_13` AggregateFunction(count), + `agg_count_14` AggregateFunction(count), + `agg_count_15` AggregateFunction(count), + `agg_count_16` AggregateFunction(count), + `agg_argmax_string_datetime_08` AggregateFunction(argMax, String, DateTime), + `agg_argmax_f32_datetime_01` AggregateFunction(argMax, Float32, DateTime), + `agg_argmax_string_datetime_09` AggregateFunction(argMax, String, DateTime), + `agg_argmax_f32_datetime_02` AggregateFunction(argMax, Float32, DateTime), + `agg_argmax_date_datetime_01` AggregateFunction(argMax, Date, DateTime), + `agg_argmax_date_datetime_02` AggregateFunction(argMax, Date, DateTime), + `agg_argmax_u8_other_01` AggregateFunction(argMax, UInt8, DateTime), + `agg_argmax_u8_other_02` AggregateFunction(argMax, UInt8, DateTime), + `agg_argmax_u8_other_03` AggregateFunction(argMax, UInt8, DateTime), + `agg_argmax_u8_other_04` AggregateFunction(argMax, UInt8, DateTime), + `agg_argmax_string_datetime_10` AggregateFunction(argMax, String, DateTime), + `agg_argmax_u8_other_05` AggregateFunction(argMax, UInt8, DateTime), + `agg_argmax_u8_other_06` AggregateFunction(argMax, UInt8, DateTime), + `agg_argmax_u8_other_07` AggregateFunction(argMax, UInt8, DateTime), + `agg_argmax_string_datetime_11` AggregateFunction(argMax, String, DateTime), + `other_max_datetime_01` SimpleAggregateFunction(max, DateTime), + `other_max_datetime_02` SimpleAggregateFunction(max, DateTime), + `nDateTime_03_date` SimpleAggregateFunction(max, DateTime), + `nDateTime_03_shown_date` SimpleAggregateFunction(max, DateTime), + `nDateTime_04_date` SimpleAggregateFunction(max, DateTime), + `nDateTime_04_shown_date` SimpleAggregateFunction(max, DateTime), + `aggCount_3` AggregateFunction(count), + `uniq_date_agg` AggregateFunction(uniq, Date), + `aggCount_4` AggregateFunction(count), + `agg_argmax_u128_datetime_01` AggregateFunction(argMax, UInt128, DateTime), + `topk_u128_01` AggregateFunction(topKWeighted(5), UInt128, UInt32), + `agg_argmax_string_datetime_12` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_datetime_13` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_datetime_14` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_datetime_15` AggregateFunction(argMax, String, DateTime), + `agg_argmax_u32_datetime_01` AggregateFunction(argMax, UInt32, DateTime), + `agg_argmax_string_datetime_16` AggregateFunction(argMax, String, DateTime), + `agg_argmax_string_u8_100` AggregateFunction(argMax, String, UInt8), + `agg_argmax_string_datetime_18` AggregateFunction(argMax, String, DateTime), + `other_max_datetime_05` SimpleAggregateFunction(max, DateTime), + `topk_Datetime_u32_u32` AggregateFunction(topKWeighted(5), UInt32, UInt32), + `agg_argmax_string_datetime_17` AggregateFunction(argMax, String, DateTime), + `other_max_datetime_09` SimpleAggregateFunction(max, DateTime), + `agg_count_17` AggregateFunction(count), + `agg_count_18` AggregateFunction(count), + `agg_count_19` AggregateFunction(count), + `agg_count_20` AggregateFunction(count) + ) + ENGINE = AggregatingMergeTree() + PARTITION BY toYYYYMM(dateColumn) + ORDER BY idColumnU64 + TTL dateColumn + toIntervalMonth(6) + SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1, min_rows_for_wide_part = 1000000000; + + DROP TABLE interpret_table_01 + + + CREATE TABLE IF NOT EXISTS interpret_table_02 + ( + `idColumnU64` UInt64, + `dateColumn` DateTime, + `agg_uniq_u128_01` AggregateFunction(uniq, UInt128), + `agg_uniq_u128_02` AggregateFunction(uniq, UInt128), + `aggCount` AggregateFunction(count), + `agg_uniq_u128_03` AggregateFunction(uniq, UInt128), + `agg_uniq_u128_04` AggregateFunction(uniq, UInt128), + `aggCount_3` AggregateFunction(count), + `aggCount_4` AggregateFunction(count), + `agg_topk_01` AggregateFunction(topKWeighted(2), UInt128, UInt64) + ) + ENGINE = AggregatingMergeTree() + PARTITION BY toYYYYMM(dateColumn) + ORDER BY idColumnU64 + TTL dateColumn + toIntervalMonth(6) + SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1; + + DROP TABLE interpret_table_02 + + + CREATE TABLE IF NOT EXISTS interpret_table_03 + ( + `idColumnU64` UInt64, + `dateColumn` Date, + `aggCount` AggregateFunction(count), + `aggCount_2` AggregateFunction(count), + `aggCount_2_shown` AggregateFunction(count), + `minDate` SimpleAggregateFunction(min, Date), + `maxDate` SimpleAggregateFunction(max, Date), + `maxInt16` SimpleAggregateFunction(max, Int16), + `minUInt16` SimpleAggregateFunction(min, UInt16), + `minUInt16_2` SimpleAggregateFunction(min, UInt16), + `aggCount_3` AggregateFunction(count), + `aggCount_4` AggregateFunction(count) + ) + ENGINE = AggregatingMergeTree() + PARTITION BY toYYYYMM(dateColumn) + ORDER BY (idColumnU64, dateColumn) + TTL dateColumn + toIntervalDay(30) + SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1; + + DROP TABLE interpret_table_03 + + + CREATE TABLE IF NOT EXISTS interpret_table_04 + ( + `idColumnU64` UInt64, + `dateColumn` DateTime, + `u128_id_02` UInt128, + `ls_01` LowCardinality(String), + `agg_count_01` AggregateFunction(count), + `agg_count_02` AggregateFunction(count), + `agg_smax_datetime_01` SimpleAggregateFunction(max, DateTime), + `agg_smax_datetime_02` SimpleAggregateFunction(max, DateTime), + `agg_count_03` AggregateFunction(count), + `agg_count_04` AggregateFunction(count) + ) + ENGINE = AggregatingMergeTree() + PARTITION BY toYYYYMM(dateColumn) + ORDER BY (idColumnU64, u128_id_02, ls_01) + TTL dateColumn + toIntervalMonth(6) + SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1; + + DROP TABLE interpret_table_04 + + + CREATE TABLE IF NOT EXISTS interpret_table_05 + ( + `idColumnU64` UInt64, + `dateColumn` Date, + `agg_uniq_u128_01` AggregateFunction(uniq, UInt128), + `agg_uniq_u128_02` AggregateFunction(uniq, UInt128), + `agg_uniq_u128_03` AggregateFunction(uniq, UInt128), + `agg_uniq_u128_04` AggregateFunction(uniq, UInt128), + `aggCount_3` AggregateFunction(count), + `aggCount_4` AggregateFunction(count), + `aggCount` AggregateFunction(count) + ) + ENGINE = AggregatingMergeTree() + PARTITION BY toYYYYMM(dateColumn) + ORDER BY (idColumnU64, dateColumn) + TTL dateColumn + toIntervalDay(30) + SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1; + + DROP TABLE interpret_table_05 + + + CREATE TABLE IF NOT EXISTS interpret_table_06 + ( + `idColumnU64` UInt64, + `dateColumn` DateTime, + `aggCount_3` AggregateFunction(count), + `aggCount` AggregateFunction(count), + `sagg_max_date` SimpleAggregateFunction(max, DateTime) + ) + ENGINE = AggregatingMergeTree() + PARTITION BY toYYYYMM(dateColumn) + ORDER BY idColumnU64 + TTL dateColumn + toIntervalMonth(6) + SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1; + + DROP TABLE interpret_table_06 + + + CREATE TABLE IF NOT EXISTS interpret_table_07 + ( + `idU128` UInt128, + `idU128_2` UInt128, + `idU128_3` UInt128, + `nI16` Nullable(Int16) DEFAULT CAST(NULL, 'Nullable(Int16)'), + `idColumnI64` Nullable(Int64) DEFAULT CAST(NULL, 'Nullable(Int64)'), + `nStr` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nStr_2` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nI16_02` Nullable(Int16) DEFAULT CAST(NULL, 'Nullable(Int16)'), + `nStr_3` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nStr_4` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nStr_5` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nI8_01` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_02` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_03` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_04` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_05` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_06` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nDate_01` Nullable(Date) DEFAULT CAST(NULL, 'Nullable(Date)'), + `nStr_6` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nStr_7` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nStr_8` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nStr_9` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nStr_10` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nStr_11` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nI8_07` Nullable(UInt8) DEFAULT CAST(NULL, 'Nullable(UInt8)'), + `nI8_08` Nullable(UInt8) DEFAULT CAST(NULL, 'Nullable(UInt8)'), + `Str_01` String, + `nI32_01` Nullable(Int32) DEFAULT CAST(NULL, 'Nullable(Int32)'), + `nI8_19` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_09` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_10` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_11` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_12` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_13` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_14` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nStr_12` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nStr_13` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nI8_15` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_16` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nDateTime_01` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_02` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_03` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_04` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_05` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_06` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_07` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_08` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_09` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_10` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_11` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nDateTime_12` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nF64_01` Nullable(Float64) DEFAULT CAST(NULL, 'Nullable(Float64)'), + `nStr_14` Nullable(String) DEFAULT CAST(NULL, 'Nullable(String)'), + `nDate_02` Nullable(Date) DEFAULT CAST(NULL, 'Nullable(Date)'), + `nDateTime_13` Nullable(DateTime) DEFAULT CAST(NULL, 'Nullable(DateTime)'), + `nF64_02` Nullable(Float64) DEFAULT CAST(NULL, 'Nullable(Float64)'), + `nF64_03` Nullable(Float64) DEFAULT CAST(NULL, 'Nullable(Float64)'), + `nF64_04` Nullable(Float64) DEFAULT CAST(NULL, 'Nullable(Float64)'), + `nF64_05` Nullable(Float64) DEFAULT CAST(NULL, 'Nullable(Float64)'), + `nI8_18` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)'), + `nI8_17` Nullable(Int8) DEFAULT CAST(NULL, 'Nullable(Int8)') + ) + ENGINE = Join(ANY, LEFT, idU128); + + DROP TABLE interpret_table_07 + + + CREATE TABLE IF NOT EXISTS interpret_table_08 + ( + `idColumnU64` UInt64, + `dateColumn` Date, + `aggCount_3` AggregateFunction(count), + `aggCount_4` AggregateFunction(count) + ) + ENGINE = AggregatingMergeTree() + PARTITION BY toYYYYMM(dateColumn) + ORDER BY (idColumnU64, dateColumn) + TTL dateColumn + toIntervalDay(30) + SETTINGS index_granularity = 1024, ttl_only_drop_parts = 1; + + DROP TABLE interpret_table_08 + + + + SELECT * + FROM + ( + SELECT + cityHash64('0321352416546546546546546546546', lower('BU'), lower('random2'), toUInt128(toUInt128('1015029'))) AS idColumnU64, + * + FROM + ( + SELECT + if(max(nDateTime_02_date_292929292) > '2020-10-31 00:00:00', max(nDateTime_02_date_292929292), NULL) AS o1, + if(max(other_max_datetime_05) > '2020-10-31 00:00:00', max(other_max_datetime_05), NULL) AS o2, + if(max(nDateTime_03_date) > '2020-10-31 00:00:00', max(nDateTime_03_date), NULL) AS o3, + if(max(nDateTime_04_date) > '2020-10-31 00:00:00', max(nDateTime_04_date), NULL) AS o4, + if(max(nDateTime_02_date) > '2020-10-31 00:00:00', max(nDateTime_02_date), NULL) AS o5, + if(max(other_max_datetime_01) > '2020-10-31 00:00:00', max(other_max_datetime_01), NULL) AS o6, + if(max(other_max_datetime_02) > '2020-10-31 00:00:00', max(other_max_datetime_02), NULL) AS o7, + argMaxMerge(agg_argmax_string_datetime_13) AS o8, + argMaxMerge(agg_argmax_string_datetime_05) AS o9, + argMaxMerge(agg_argmax_string_datetime_06) AS o10, + argMaxMerge(agg_argmax_string_datetime_02) AS o11, + argMaxMerge(agg_argmax_string_datetime_04) AS o12, + argMaxMerge(agg_argmax_string_datetime_15) AS o13, + argMaxMerge(agg_argmax_string_datetime_01) AS o14, + argMaxMerge(agg_argmax_string_u8_01) AS o15, + argMaxMerge(agg_argmax_f32_datetime_02) AS o16, + if(argMaxMerge(agg_argmax_string_datetime_09) != '', argMaxMerge(agg_argmax_string_datetime_09), NULL) AS o17, + if(argMaxMerge(agg_argmax_date_datetime_01) > '2020-10-31', argMaxMerge(agg_argmax_date_datetime_01), NULL) AS o18, + if(argMaxMerge(agg_argmax_date_datetime_02) > '2020-10-31', argMaxMerge(agg_argmax_date_datetime_02), NULL) AS o19, + argMaxMerge(agg_argmax_u8_other_02) AS o20, + argMaxMerge(agg_argmax_u8_other_03) AS o21, + argMaxMerge(agg_argmax_u8_other_04) AS o22, + argMaxMerge(agg_argmax_u8_other_01) AS o23, + argMaxMerge(agg_argmax_string_datetime_10) AS o24, + argMaxMerge(agg_argmax_string_datetime_11) AS o25, + countMerge(aggCount_3) AS o26, + countMerge(aggCount_4) AS o27 + FROM interpret_table_01 AS c + PREWHERE cityHash64('0321352416546546546546546546546', lower('BU'), lower('random2'), toUInt128(toUInt128('1015029'))) = c.idColumnU64 + ) AS s01, + ( + WITH ( + SELECT coalesce(if((topKWeightedMerge(2)(agg_topk_01)[1]) != toUInt128(toUInt128('1015029')), topKWeightedMerge(2)(agg_topk_01)[1], topKWeightedMerge(2)(agg_topk_01)[2]), 0) + FROM interpret_table_02 AS c + PREWHERE cityHash64('0321352416546546546546546546546', lower('BU'), lower('random2')) = c.idColumnU64 + ) AS other_idU128 + SELECT + if(max(other_max_datetime_05) > '2020-10-31 00:00:00', max(other_max_datetime_05), NULL) AS o28, + if(max(other_max_datetime_01) > '2020-10-31 00:00:00', max(other_max_datetime_01), NULL) AS o29, + if(max(nDateTime_02_date) > '2020-10-31 00:00:00', max(nDateTime_02_date), NULL) AS o30, + other_idU128 + FROM interpret_table_01 AS c + PREWHERE cityHash64('0321352416546546546546546546546', lower('BU'), lower('random2'), toUInt128(other_idU128)) = c.idColumnU64 + ) AS s02, + ( + SELECT + minIf(minDate, dateColumn > (now() - toIntervalDay(7))) AS o31, + maxIf(maxDate, dateColumn > (now() - toIntervalDay(7))) AS o32, + maxIf(maxInt16, dateColumn > (now() - toIntervalDay(28))) AS o33, + countMergeIf(aggCount_3, dateColumn > (now() - toIntervalHour(24))) AS o34, + countMergeIf(aggCount_3, dateColumn > (now() - toIntervalDay(14))) AS o35, + countMergeIf(aggCount_3, dateColumn > (now() - toIntervalDay(28))) AS o36, + countMergeIf(aggCount_4, dateColumn > (now() - toIntervalHour(24))) AS o37, + countMergeIf(aggCount_4, dateColumn > (now() - toIntervalDay(7))) AS o38, + countMergeIf(aggCount_4, dateColumn > (now() - toIntervalDay(28))) AS o27_month, + countMergeIf(aggCount_2_shown, dateColumn > (now() - toIntervalDay(14))) AS o40 + FROM interpret_table_03 AS c + PREWHERE cityHash64('0321352416546546546546546546546', lower('BU'), lower('random2'), toUInt128(toUInt128('1015029'))) = c.idColumnU64 + ) AS s03, + ( + SELECT + countMerge(agg_count_03) AS o41, + countMerge(agg_count_04) AS o42 + FROM interpret_table_04 AS c + PREWHERE (cityHash64('0321352416546546546546546546546', lower('BU'), lower('random2'), toUInt128(toUInt128('1015029'))) = c.idColumnU64) AND (ls_01 = 'exit') + ) AS s04, + ( + SELECT + countMerge(aggCount_3) AS o43, + countMerge(aggCount_4) AS o44, + countMerge(aggCount) AS o45 + FROM interpret_table_02 AS c + PREWHERE cityHash64('0321352416546546546546546546546', lower('BU'), lower('random2')) = c.idColumnU64 + ) AS s05, + ( + SELECT + countMergeIf(aggCount_3, dateColumn > (now() - toIntervalDay(14))) AS o46, + uniqMergeIf(agg_uniq_u128_03, dateColumn > (now() - toIntervalHour(24))) AS o47, + uniqMergeIf(agg_uniq_u128_03, dateColumn > (now() - toIntervalDay(14))) AS o48, + countMergeIf(aggCount_4, dateColumn > (now() - toIntervalDay(14))) AS o49, + countMergeIf(aggCount_4, dateColumn > (now() - toIntervalDay(28))) AS o50 + FROM interpret_table_05 AS c + PREWHERE cityHash64('0321352416546546546546546546546', lower('BU'), lower('random2')) = c.idColumnU64 + ) AS s06, + ( + SELECT countMerge(aggCount_3) AS o51 + FROM interpret_table_06 AS c + PREWHERE cityHash64('0321352416546546546546546546546', lower('BU'), lower('random2'), toUInt128(joinGet(interpret_table_07, 'idColumnI64', toUInt128('1015029')))) = c.idColumnU64 + ) AS s07, + ( + SELECT + countMergeIf(aggCount_3, dateColumn > (now() - toIntervalDay(28))) AS s52, + countMergeIf(aggCount_4, dateColumn > (now() - toIntervalDay(28))) AS s53 + FROM interpret_table_08 AS c + PREWHERE cityHash64('0321352416546546546546546546546', lower('BU'), lower('random2'), toUInt128(joinGet(interpret_table_07, 'idColumnI64', toUInt128('1015029')))) = c.idColumnU64 + ) AS s08 + ) AS final_s01 + FORMAT JSONEachRow; + + diff --git a/tests/queries/0_stateless/00502_sum_map.reference b/tests/queries/0_stateless/00502_sum_map.reference index 31b067a2bc9..b1cd0303004 100644 --- a/tests/queries/0_stateless/00502_sum_map.reference +++ b/tests/queries/0_stateless/00502_sum_map.reference @@ -1,26 +1,70 @@ +-- { echoOn } +DROP TABLE IF EXISTS sum_map; +CREATE TABLE sum_map(date Date, timeslot DateTime, statusMap Nested(status UInt16, requests UInt64)) ENGINE = Log; +INSERT INTO sum_map VALUES ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]), ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]), ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]), ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]); +SELECT * FROM sum_map ORDER BY timeslot, statusMap.status, statusMap.requests; 2000-01-01 2000-01-01 00:00:00 [1,2,3] [10,10,10] 2000-01-01 2000-01-01 00:00:00 [3,4,5] [10,10,10] 2000-01-01 2000-01-01 00:01:00 [4,5,6] [10,10,10] 2000-01-01 2000-01-01 00:01:00 [6,7,8] [10,10,10] +SELECT sumMap(statusMap.status, statusMap.requests) FROM sum_map; ([1,2,3,4,5,6,7,8],[10,10,20,20,20,20,10,10]) +SELECT sumMap((statusMap.status, statusMap.requests)) FROM sum_map; ([1,2,3,4,5,6,7,8],[10,10,20,20,20,20,10,10]) +SELECT sumMapMerge(s) FROM (SELECT sumMapState(statusMap.status, statusMap.requests) AS s FROM sum_map); ([1,2,3,4,5,6,7,8],[10,10,20,20,20,20,10,10]) +SELECT timeslot, sumMap(statusMap.status, statusMap.requests) FROM sum_map GROUP BY timeslot ORDER BY timeslot; 2000-01-01 00:00:00 ([1,2,3,4,5],[10,10,20,10,10]) 2000-01-01 00:01:00 ([4,5,6,7,8],[10,10,20,10,10]) +SELECT timeslot, sumMap(statusMap.status, statusMap.requests).1, sumMap(statusMap.status, statusMap.requests).2 FROM sum_map GROUP BY timeslot ORDER BY timeslot; 2000-01-01 00:00:00 [1,2,3,4,5] [10,10,20,10,10] 2000-01-01 00:01:00 [4,5,6,7,8] [10,10,20,10,10] +SELECT sumMapFiltered([1])(statusMap.status, statusMap.requests) FROM sum_map; ([1],[10]) +SELECT sumMapFiltered([1, 4, 8])(statusMap.status, statusMap.requests) FROM sum_map; ([1,4,8],[10,20,10]) +DROP TABLE sum_map; +DROP TABLE IF EXISTS sum_map_overflow; +CREATE TABLE sum_map_overflow(events Array(UInt8), counts Array(UInt8)) ENGINE = Log; +INSERT INTO sum_map_overflow VALUES ([1], [255]), ([1], [2]); +SELECT sumMap(events, counts) FROM sum_map_overflow; ([1],[257]) +SELECT sumMapWithOverflow(events, counts) FROM sum_map_overflow; ([1],[1]) +DROP TABLE sum_map_overflow; +select sumMap(val, cnt) from ( SELECT [ CAST(1, 'UInt64') ] as val, [1] as cnt ); ([1],[1]) +select sumMap(val, cnt) from ( SELECT [ CAST(1, 'Float64') ] as val, [1] as cnt ); ([1],[1]) +select sumMap(val, cnt) from ( SELECT [ CAST('a', 'Enum16(\'a\'=1)') ] as val, [1] as cnt ); (['a'],[1]) +select sumMap(val, cnt) from ( SELECT [ CAST(1, 'DateTime(\'Asia/Istanbul\')') ] as val, [1] as cnt ); (['1970-01-01 02:00:01'],[1]) +select sumMap(val, cnt) from ( SELECT [ CAST(1, 'Date') ] as val, [1] as cnt ); (['1970-01-02'],[1]) +select sumMap(val, cnt) from ( SELECT [ CAST('01234567-89ab-cdef-0123-456789abcdef', 'UUID') ] as val, [1] as cnt ); (['01234567-89ab-cdef-0123-456789abcdef'],[1]) +select sumMap(val, cnt) from ( SELECT [ CAST(1.01, 'Decimal(10,2)') ] as val, [1] as cnt ); ([1.01],[1]) +select sumMap(val, cnt) from ( SELECT [ CAST('a', 'FixedString(1)'), CAST('b', 'FixedString(1)' ) ] as val, [1, 2] as cnt ); (['a','b'],[1,2]) +select sumMap(val, cnt) from ( SELECT [ CAST('abc', 'String'), CAST('ab', 'String'), CAST('a', 'String') ] as val, [1, 2, 3] as cnt ); (['a','ab','abc'],[3,2,1]) +DROP TABLE IF EXISTS sum_map_decimal; +CREATE TABLE sum_map_decimal( + statusMap Nested( + goal_id UInt16, + revenue Decimal32(5) + ) +) ENGINE = Log; +INSERT INTO sum_map_decimal VALUES ([1, 2, 3], [1.0, 2.0, 3.0]), ([3, 4, 5], [3.0, 4.0, 5.0]), ([4, 5, 6], [4.0, 5.0, 6.0]), ([6, 7, 8], [6.0, 7.0, 8.0]); +SELECT sumMap(statusMap.goal_id, statusMap.revenue) FROM sum_map_decimal; ([1,2,3,4,5,6,7,8],[1,2,6,8,10,12,7,8]) +SELECT sumMapWithOverflow(statusMap.goal_id, statusMap.revenue) FROM sum_map_decimal; ([1,2,3,4,5,6,7,8],[1,2,6,8,10,12,7,8]) +DROP TABLE sum_map_decimal; +CREATE TABLE sum_map_decimal_nullable (`statusMap` Array(Tuple(goal_id UInt16, revenue Nullable(Decimal(9, 5))))) engine=Log; +INSERT INTO sum_map_decimal_nullable VALUES ([1, 2, 3], [1.0, 2.0, 3.0]), ([3, 4, 5], [3.0, 4.0, 5.0]), ([4, 5, 6], [4.0, 5.0, 6.0]), ([6, 7, 8], [6.0, 7.0, 8.0]); +SELECT sumMap(statusMap.goal_id, statusMap.revenue) FROM sum_map_decimal_nullable; +([1,2,3,4,5,6,7,8],[1,2,6,8,10,12,7,8]) +DROP TABLE sum_map_decimal_nullable; diff --git a/tests/queries/0_stateless/00502_sum_map.sql b/tests/queries/0_stateless/00502_sum_map.sql index acc87cc5f16..30037d49784 100644 --- a/tests/queries/0_stateless/00502_sum_map.sql +++ b/tests/queries/0_stateless/00502_sum_map.sql @@ -1,5 +1,6 @@ SET send_logs_level = 'fatal'; +-- { echoOn } DROP TABLE IF EXISTS sum_map; CREATE TABLE sum_map(date Date, timeslot DateTime, statusMap Nested(status UInt16, requests UInt64)) ENGINE = Log; @@ -54,3 +55,8 @@ SELECT sumMap(statusMap.goal_id, statusMap.revenue) FROM sum_map_decimal; SELECT sumMapWithOverflow(statusMap.goal_id, statusMap.revenue) FROM sum_map_decimal; DROP TABLE sum_map_decimal; + +CREATE TABLE sum_map_decimal_nullable (`statusMap` Array(Tuple(goal_id UInt16, revenue Nullable(Decimal(9, 5))))) engine=Log; +INSERT INTO sum_map_decimal_nullable VALUES ([1, 2, 3], [1.0, 2.0, 3.0]), ([3, 4, 5], [3.0, 4.0, 5.0]), ([4, 5, 6], [4.0, 5.0, 6.0]), ([6, 7, 8], [6.0, 7.0, 8.0]); +SELECT sumMap(statusMap.goal_id, statusMap.revenue) FROM sum_map_decimal_nullable; +DROP TABLE sum_map_decimal_nullable; diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql index e1392d299dc..bbc7bedcb4f 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql @@ -12,7 +12,7 @@ CREATE TABLE check_system_tables ORDER BY name1 PARTITION BY name2 SAMPLE BY name1 - SETTINGS min_bytes_for_wide_part = 0; + SETTINGS min_bytes_for_wide_part = 0, compress_marks=false, compress_primary_key=false; SELECT name, partition_key, sorting_key, primary_key, sampling_key, storage_policy, total_rows FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase() @@ -36,7 +36,8 @@ CREATE TABLE check_system_tables sign Int8 ) ENGINE = VersionedCollapsingMergeTree(sign, version) PARTITION BY date - ORDER BY date; + ORDER BY date + SETTINGS compress_marks=false, compress_primary_key=false; SELECT name, partition_key, sorting_key, primary_key, sampling_key FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase() diff --git a/tests/queries/0_stateless/00800_low_cardinality_merge_join.reference.j2 b/tests/queries/0_stateless/00800_low_cardinality_merge_join.reference.j2 index 06001261088..296e0276653 100644 --- a/tests/queries/0_stateless/00800_low_cardinality_merge_join.reference.j2 +++ b/tests/queries/0_stateless/00800_low_cardinality_merge_join.reference.j2 @@ -1,4 +1,4 @@ -{% for join_algorithm in ['partial_merge', 'full_sorting_merge'] -%} +{% for join_algorithm in ['partial_merge', 'full_sorting_merge', 'grace_hash'] -%} 0 0 0 diff --git a/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql.j2 b/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql.j2 index d0dd908ae67..8b7856b7738 100644 --- a/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql.j2 +++ b/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql.j2 @@ -1,34 +1,34 @@ -{% for join_algorithm in ['partial_merge', 'full_sorting_merge'] -%} +{% for join_algorithm in ['partial_merge', 'full_sorting_merge', 'grace_hash'] -%} -set join_algorithm = '{{ join_algorithm }}'; +SET join_algorithm = '{{ join_algorithm }}'; -select * from (select dummy as val from system.one) s1 any left join (select dummy as val from system.one) s2 using val; -select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select dummy as val from system.one) s2 using val; -select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(dummy) as val from system.one) s2 using val; -select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(dummy) as val from system.one) s2 using val; -select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select dummy as val from system.one) s2 using val; -select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as val from system.one) s2 using val; -select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(dummy) as val from system.one) s2 using val; -select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as val from system.one) s2 using val; -select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as val from system.one) s2 using val; -select '-'; -select * from (select dummy as val from system.one) s1 any left join (select dummy as val from system.one) s2 on val + 0 = val * 1; -- { serverError 352 } -select * from (select dummy as val from system.one) s1 any left join (select dummy as rval from system.one) s2 on val + 0 = rval * 1; -select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select dummy as rval from system.one) s2 on val + 0 = rval * 1; -select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(dummy) as rval from system.one) s2 on val + 0 = rval * 1; -select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(dummy) as rval from system.one) s2 on val + 0 = rval * 1; -select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select dummy as rval from system.one) s2 on val + 0 = rval * 1; -select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as rval from system.one) s2 on val + 0 = rval * 1; -select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(dummy) as rval from system.one) s2 on val + 0 = rval * 1; -select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as rval from system.one) s2 on val + 0 = rval * 1; -select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as rval from system.one) s2 on val + 0 = rval * 1; -select '-'; -select * from (select number as l from system.numbers limit 3) s1 any left join (select number as r from system.numbers limit 3) s2 on l + 1 = r * 1; -select * from (select toLowCardinality(number) as l from system.numbers limit 3) s1 any left join (select number as r from system.numbers limit 3) s2 on l + 1 = r * 1; -select * from (select number as l from system.numbers limit 3) s1 any left join (select toLowCardinality(number) as r from system.numbers limit 3) s2 on l + 1 = r * 1; -select * from (select toLowCardinality(number) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(number) as r from system.numbers limit 3) s2 on l + 1 = r * 1; -select * from (select toLowCardinality(toNullable(number)) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(number) as r from system.numbers limit 3) s2 on l + 1 = r * 1; -select * from (select toLowCardinality(number) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(toNullable(number)) as r from system.numbers limit 3) s2 on l + 1 = r * 1; -select * from (select toLowCardinality(toNullable(number)) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(toNullable(number)) as r from system.numbers limit 3) s2 on l + 1 = r * 1; +SELECT * FROM (SELECT dummy AS val FROM system.one) s1 ANY LEFT JOIN (SELECT dummy AS val FROM system.one) s2 USING val ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(dummy) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT dummy AS val FROM system.one) s2 USING val ORDER BY val; +SELECT * FROM (SELECT dummy AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(dummy) AS val FROM system.one) s2 USING val ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(dummy) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(dummy) AS val FROM system.one) s2 USING val ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(toNullable(dummy)) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT dummy AS val FROM system.one) s2 USING val ORDER BY val; +SELECT * FROM (SELECT dummy AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(toNullable(dummy)) AS val FROM system.one) s2 USING val ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(toNullable(dummy)) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(dummy) AS val FROM system.one) s2 USING val ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(dummy) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(toNullable(dummy)) AS val FROM system.one) s2 USING val ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(toNullable(dummy)) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(toNullable(dummy)) AS val FROM system.one) s2 USING val ORDER BY val; +SELECT '-'; +SELECT * FROM (SELECT dummy AS val FROM system.one) s1 ANY LEFT JOIN (SELECT dummy AS val FROM system.one) s2 ON val + 0 = val * 1 ORDER BY val; -- { serverError 352 } +SELECT * FROM (SELECT dummy AS val FROM system.one) s1 ANY LEFT JOIN (SELECT dummy AS rval FROM system.one) s2 ON val + 0 = rval * 1 ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(dummy) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT dummy AS rval FROM system.one) s2 ON val + 0 = rval * 1 ORDER BY val; +SELECT * FROM (SELECT dummy AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(dummy) AS rval FROM system.one) s2 ON val + 0 = rval * 1 ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(dummy) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(dummy) AS rval FROM system.one) s2 ON val + 0 = rval * 1 ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(toNullable(dummy)) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT dummy AS rval FROM system.one) s2 ON val + 0 = rval * 1 ORDER BY val; +SELECT * FROM (SELECT dummy AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(toNullable(dummy)) AS rval FROM system.one) s2 ON val + 0 = rval * 1 ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(toNullable(dummy)) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(dummy) AS rval FROM system.one) s2 ON val + 0 = rval * 1 ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(dummy) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(toNullable(dummy)) AS rval FROM system.one) s2 ON val + 0 = rval * 1 ORDER BY val; +SELECT * FROM (SELECT toLowCardinality(toNullable(dummy)) AS val FROM system.one) s1 ANY LEFT JOIN (SELECT toLowCardinality(toNullable(dummy)) AS rval FROM system.one) s2 ON val + 0 = rval * 1 ORDER BY val; +SELECT '-'; +SELECT * FROM (SELECT number AS l FROM system.numbers LIMIT 3) s1 ANY LEFT JOIN (SELECT number AS r FROM system.numbers LIMIT 3) s2 ON l + 1 = r * 1 ORDER BY l; +SELECT * FROM (SELECT toLowCardinality(number) AS l FROM system.numbers LIMIT 3) s1 ANY LEFT JOIN (SELECT number AS r FROM system.numbers LIMIT 3) s2 ON l + 1 = r * 1 ORDER BY l; +SELECT * FROM (SELECT number AS l FROM system.numbers LIMIT 3) s1 ANY LEFT JOIN (SELECT toLowCardinality(number) AS r FROM system.numbers LIMIT 3) s2 ON l + 1 = r * 1 ORDER BY l; +SELECT * FROM (SELECT toLowCardinality(number) AS l FROM system.numbers LIMIT 3) s1 ANY LEFT JOIN (SELECT toLowCardinality(number) AS r FROM system.numbers LIMIT 3) s2 ON l + 1 = r * 1 ORDER BY l; +SELECT * FROM (SELECT toLowCardinality(toNullable(number)) AS l FROM system.numbers LIMIT 3) s1 ANY LEFT JOIN (SELECT toLowCardinality(number) AS r FROM system.numbers LIMIT 3) s2 ON l + 1 = r * 1 ORDER BY l; +SELECT * FROM (SELECT toLowCardinality(number) AS l FROM system.numbers LIMIT 3) s1 ANY LEFT JOIN (SELECT toLowCardinality(toNullable(number)) AS r FROM system.numbers LIMIT 3) s2 ON l + 1 = r * 1 ORDER BY l; +SELECT * FROM (SELECT toLowCardinality(toNullable(number)) AS l FROM system.numbers LIMIT 3) s1 ANY LEFT JOIN (SELECT toLowCardinality(toNullable(number)) AS r FROM system.numbers LIMIT 3) s2 ON l + 1 = r * 1 ORDER BY l; {% endfor -%} diff --git a/tests/queries/0_stateless/00804_test_delta_codec_compression.sql b/tests/queries/0_stateless/00804_test_delta_codec_compression.sql index 044d60aeafb..ca9bb1b177e 100644 --- a/tests/queries/0_stateless/00804_test_delta_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_delta_codec_compression.sql @@ -9,12 +9,12 @@ DROP TABLE IF EXISTS default_codec_synthetic; CREATE TABLE delta_codec_synthetic ( id UInt64 Codec(Delta, ZSTD(3)) -) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key=false; CREATE TABLE default_codec_synthetic ( id UInt64 Codec(ZSTD(3)) -) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key=false; INSERT INTO delta_codec_synthetic SELECT number FROM system.numbers LIMIT 5000000; INSERT INTO default_codec_synthetic SELECT number FROM system.numbers LIMIT 5000000; @@ -47,12 +47,12 @@ DROP TABLE IF EXISTS default_codec_float; CREATE TABLE delta_codec_float ( id Float64 Codec(Delta, LZ4HC) -) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key=false; CREATE TABLE default_codec_float ( id Float64 Codec(LZ4HC) -) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key=false; INSERT INTO delta_codec_float SELECT number FROM numbers(1547510400, 500000) WHERE number % 3 == 0 OR number % 5 == 0 OR number % 7 == 0 OR number % 11 == 0; INSERT INTO default_codec_float SELECT * from delta_codec_float; @@ -85,12 +85,12 @@ DROP TABLE IF EXISTS default_codec_string; CREATE TABLE delta_codec_string ( id Float64 Codec(Delta, LZ4) -) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key=false; CREATE TABLE default_codec_string ( id Float64 Codec(LZ4) -) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key=false; INSERT INTO delta_codec_string SELECT concat(toString(number), toString(number % 100)) FROM numbers(1547510400, 500000); INSERT INTO default_codec_string SELECT * from delta_codec_string; diff --git a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql index 792bf62f9b1..cf9fd3cad12 100644 --- a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql +++ b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS test_00961; CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) - ENGINE = MergeTree PARTITION BY d ORDER BY (a, b) SETTINGS index_granularity = 111, min_bytes_for_wide_part = 0; + ENGINE = MergeTree PARTITION BY d ORDER BY (a, b) SETTINGS index_granularity = 111, min_bytes_for_wide_part = 0, compress_marks=false, compress_primary_key=false; INSERT INTO test_00961 VALUES ('2000-01-01', 'Hello, world!', 123, 'xxx yyy', -123, 123456789); diff --git a/tests/queries/0_stateless/01010_partial_merge_join_const_and_lc.reference b/tests/queries/0_stateless/01010_partial_merge_join_const_and_lc.reference index 95859e3e0a4..0ace422adc2 100644 --- a/tests/queries/0_stateless/01010_partial_merge_join_const_and_lc.reference +++ b/tests/queries/0_stateless/01010_partial_merge_join_const_and_lc.reference @@ -8,3 +8,8 @@ 3 4 5 +1 1 +2 +3 +4 +5 diff --git a/tests/queries/0_stateless/01010_partial_merge_join_const_and_lc.sql b/tests/queries/0_stateless/01010_partial_merge_join_const_and_lc.sql index 9abfc425d83..51559897120 100644 --- a/tests/queries/0_stateless/01010_partial_merge_join_const_and_lc.sql +++ b/tests/queries/0_stateless/01010_partial_merge_join_const_and_lc.sql @@ -13,3 +13,11 @@ select * from (select materialize(2) as x) s1 left join (select 2 as x) s2 using select * from (select 3 as x) s1 left join (select materialize(3) as x) s2 using x; select * from (select toLowCardinality(4) as x) s1 left join (select 4 as x) s2 using x; select * from (select 5 as x) s1 left join (select toLowCardinality(5) as x) s2 using x; + +SET join_algorithm = 'grace_hash'; + +select s1.x, s2.x from (select 1 as x) s1 left join (select 1 as x) s2 using x; +select * from (select materialize(2) as x) s1 left join (select 2 as x) s2 using x; +select * from (select 3 as x) s1 left join (select materialize(3) as x) s2 using x; +select * from (select toLowCardinality(4) as x) s1 left join (select 4 as x) s2 using x; +select * from (select 5 as x) s1 left join (select toLowCardinality(5) as x) s2 using x; diff --git a/tests/queries/0_stateless/01010_pmj_on_disk.reference b/tests/queries/0_stateless/01010_pmj_on_disk.reference index ba1d03fcc5d..74f12daa203 100644 --- a/tests/queries/0_stateless/01010_pmj_on_disk.reference +++ b/tests/queries/0_stateless/01010_pmj_on_disk.reference @@ -14,3 +14,7 @@ 1 0 2 11 3 0 +0 10 +1 0 +2 11 +3 0 diff --git a/tests/queries/0_stateless/01010_pmj_on_disk.sql b/tests/queries/0_stateless/01010_pmj_on_disk.sql index 28bc0ced3b7..d4fb9184896 100644 --- a/tests/queries/0_stateless/01010_pmj_on_disk.sql +++ b/tests/queries/0_stateless/01010_pmj_on_disk.sql @@ -5,7 +5,8 @@ ANY LEFT JOIN ( SELECT number * 2 AS n, number + 10 AS j FROM numbers(4000) ) js2 -USING n; +USING n +ORDER BY n; SET max_rows_in_join = 1000; @@ -14,7 +15,8 @@ ANY LEFT JOIN ( SELECT number * 2 AS n, number + 10 AS j FROM numbers(4000) ) js2 -USING n; -- { serverError 191 } +USING n +ORDER BY n; -- { serverError 191 } SET join_algorithm = 'partial_merge'; @@ -23,7 +25,8 @@ ANY LEFT JOIN ( SELECT number * 2 AS n, number + 10 AS j FROM numbers(4000) ) js2 -USING n; +USING n +ORDER BY n; SET partial_merge_join_optimizations = 1; @@ -32,7 +35,8 @@ ANY LEFT JOIN ( SELECT number * 2 AS n, number + 10 AS j FROM numbers(4000) ) js2 -USING n; +USING n +ORDER BY n; SET join_algorithm = 'auto'; @@ -41,4 +45,15 @@ ANY LEFT JOIN ( SELECT number * 2 AS n, number + 10 AS j FROM numbers(4000) ) js2 -USING n; +USING n +ORDER BY n; + +SET max_rows_in_join = '10'; + +SELECT number as n, j FROM numbers(4) nums +ANY LEFT JOIN ( + SELECT number * 2 AS n, number + 10 AS j + FROM numbers(4000) +) js2 +USING n +ORDER BY n; diff --git a/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh b/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh index a95029de257..983cb515d8e 100755 --- a/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh +++ b/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh @@ -16,7 +16,7 @@ function create_db() # So CREATE TABLE queries will fail on all replicas except one. But it's still makes sense for a stress test. $CLICKHOUSE_CLIENT --allow_experimental_database_replicated=1 --query \ "create database if not exists ${CLICKHOUSE_DATABASE}_repl_$SUFFIX engine=Replicated('/test/01111/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '$SHARD', '$REPLICA')" \ - 2>&1| grep -Fa "Exception: " | grep -Fv "REPLICA_IS_ALREADY_EXIST" | grep -Fiv "Will not try to start it up" | \ + 2>&1| grep -Fa "Exception: " | grep -Fv "REPLICA_ALREADY_EXISTS" | grep -Fiv "Will not try to start it up" | \ grep -Fv "Coordination::Exception" | grep -Fv "already contains some data and it does not look like Replicated database path" sleep 0.$RANDOM done diff --git a/tests/queries/0_stateless/01120_join_constants.sql b/tests/queries/0_stateless/01120_join_constants.sql index d6d6a1be43b..fdf297f5934 100644 --- a/tests/queries/0_stateless/01120_join_constants.sql +++ b/tests/queries/0_stateless/01120_join_constants.sql @@ -14,7 +14,7 @@ LEFT JOIN SELECT arrayJoin([1, 3]) AS k, 'world' -) AS t2 ON t1.k = t2.k; +) AS t2 ON t1.k = t2.k ORDER BY t1.k; SELECT t1.*, @@ -32,4 +32,4 @@ LEFT JOIN SELECT arrayJoin([1, 3]) AS k, 123 -) AS t2 ON t1.k = t2.k; +) AS t2 ON t1.k = t2.k ORDER BY t1.k; diff --git a/tests/queries/0_stateless/01130_in_memory_parts_partitons.reference b/tests/queries/0_stateless/01130_in_memory_parts_partitons.reference index b9daa88b4ca..44cbbed3f57 100644 --- a/tests/queries/0_stateless/01130_in_memory_parts_partitons.reference +++ b/tests/queries/0_stateless/01130_in_memory_parts_partitons.reference @@ -2,35 +2,59 @@ 1 3 bar 2 4 aa 2 5 bb -3 6 qq -3 7 ww -================== +2 6 cc +3 7 qq +3 8 ww +3 9 ee +3 10 rr +1_1_1_0 InMemory 2 +2_2_2_0 InMemory 3 +3_3_3_0 InMemory 4 +^ init ================== 2 4 aa 2 5 bb -3 6 qq -3 7 ww -================== -3 6 qq -3 7 ww -================== +2 6 cc +3 7 qq +3 8 ww +3 9 ee +3 10 rr +2_2_2_0 InMemory 3 +3_3_3_0 InMemory 4 +^ drop 1 ================== +3 7 qq +3 8 ww +3 9 ee +3 10 rr +3_3_3_0 InMemory 4 +^ detach 2 ================== 2 4 aa 2 5 bb -3 6 qq -3 7 ww -2_4_4_0 Compact -3_3_3_0 InMemory -================== +2 6 cc +3 7 qq +3 8 ww +3 9 ee +3 10 rr +2_4_4_0 Compact 3 +3_3_3_0 InMemory 4 +^ attach 2 ================= 2 4 aa 2 5 bb -3 6 qq -3 7 ww -================== +2 6 cc +3 7 qq +3 8 ww +3 9 ee +3 10 rr +2_4_4_0 Compact 3 +3_3_3_0 InMemory 4 +^ detach attach ================== 2 4 aa 2 5 bb -3 6 cc -3 7 dd -t2 2_4_4_0 Compact -t2 3_6_6_0 Compact -t3 3_1_1_0 InMemory -================== -3_1_1_0 InMemory 1 +2 6 cc +3 11 tt +3 12 yy +t2 2_4_4_0 Compact 3 +t2 3_6_6_0 Compact 2 +t3 3_1_1_0 InMemory 2 +^ replace ================== +3_1_1_0 InMemory 1 2 +^ freeze ================== diff --git a/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql b/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql index aa6f281e0eb..b1ba8bc5560 100644 --- a/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql +++ b/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql @@ -9,30 +9,34 @@ CREATE TABLE t2(id UInt32, a UInt64, s String) SYSTEM STOP MERGES t2; INSERT INTO t2 VALUES (1, 2, 'foo'), (1, 3, 'bar'); -INSERT INTO t2 VALUES (2, 4, 'aa'), (2, 5, 'bb'); -INSERT INTO t2 VALUES (3, 6, 'qq'), (3, 7, 'ww'); +INSERT INTO t2 VALUES (2, 4, 'aa'), (2, 5, 'bb'), (2, 6, 'cc'); +INSERT INTO t2 VALUES (3, 7, 'qq'), (3, 8, 'ww'), (3, 9, 'ee'), (3, 10, 'rr'); SELECT * FROM t2 ORDER BY a; -SELECT '=================='; +SELECT name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name; +SELECT '^ init =================='; ALTER TABLE t2 DROP PARTITION 1; SELECT * FROM t2 ORDER BY a; -SELECT '=================='; +SELECT name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name; +SELECT '^ drop 1 =================='; ALTER TABLE t2 DETACH PARTITION 2; SELECT * FROM t2 ORDER BY a; -SELECT '=================='; +SELECT name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name; +SELECT '^ detach 2 =================='; ALTER TABLE t2 ATTACH PARTITION 2; SELECT * FROM t2 ORDER BY a; -SELECT name, part_type FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name; -SELECT '=================='; +SELECT name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name; +SELECT '^ attach 2 ================='; DETACH TABLE t2; ATTACH TABLE t2; SELECT * FROM t2 ORDER BY a; -SELECT '=================='; +SELECT name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name; +SELECT '^ detach attach =================='; DROP TABLE IF EXISTS t3; @@ -40,15 +44,16 @@ CREATE TABLE t3(id UInt32, a UInt64, s String) ENGINE = MergeTree ORDER BY a PARTITION BY id SETTINGS min_rows_for_compact_part = 1000, min_rows_for_wide_part = 2000; -INSERT INTO t3 VALUES (3, 6, 'cc'), (3, 7, 'dd'); +INSERT INTO t3 VALUES (3, 11, 'tt'), (3, 12, 'yy'); ALTER TABLE t2 REPLACE PARTITION 3 FROM t3; SELECT * FROM t2 ORDER BY a; -SELECT table, name, part_type FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name; -SELECT table, name, part_type FROM system.parts WHERE table = 't3' AND active AND database = currentDatabase() ORDER BY name; -SELECT '=================='; +SELECT table, name, part_type, rows FROM system.parts WHERE table = 't2' AND active AND database = currentDatabase() ORDER BY name; +SELECT table, name, part_type, rows FROM system.parts WHERE table = 't3' AND active AND database = currentDatabase() ORDER BY name; +SELECT '^ replace =================='; ALTER TABLE t3 FREEZE PARTITION 3; -SELECT name, part_type, is_frozen FROM system.parts WHERE table = 't3' AND active AND database = currentDatabase() ORDER BY name; +SELECT name, part_type, is_frozen, rows FROM system.parts WHERE table = 't3' AND active AND database = currentDatabase() ORDER BY name; +SELECT '^ freeze =================='; DROP TABLE t2; DROP TABLE t3; diff --git a/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql b/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql index ae844888407..d73d438d9da 100644 --- a/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql +++ b/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql @@ -17,7 +17,7 @@ SELECT t1.id, t2.id as id, t3.id as value FROM (select number as id, 42 as value from numbers(4)) t1 LEFT JOIN (select number as id, 42 as value from numbers(3)) t2 ON t1.id = t2.id LEFT JOIN (select number as id, 42 as value from numbers(2)) t3 ON t1.id = t3.id -WHERE id > 0 AND value < 42; +WHERE id > 0 AND value < 42 ORDER BY id; CREATE VIEW IF NOT EXISTS view1 AS SELECT t1.id AS id, t1.value1 AS value1, t2.value2 AS value2, t3.value3 AS value3 @@ -26,7 +26,7 @@ CREATE VIEW IF NOT EXISTS view1 AS LEFT JOIN t3 ON t1.id = t3.id WHERE t1.id > 0; -SELECT * FROM view1 WHERE id = 1; +SELECT * FROM view1 WHERE id = 1 ORDER BY id; DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; diff --git a/tests/queries/0_stateless/01155_rename_move_materialized_view.sql b/tests/queries/0_stateless/01155_rename_move_materialized_view.sql index b3234e03a8f..c3cc0bbb9eb 100644 --- a/tests/queries/0_stateless/01155_rename_move_materialized_view.sql +++ b/tests/queries/0_stateless/01155_rename_move_materialized_view.sql @@ -39,7 +39,7 @@ RENAME TABLE test_01155_ordinary.mv1 TO test_01155_atomic.mv1; RENAME TABLE test_01155_ordinary.mv2 TO test_01155_atomic.mv2; RENAME TABLE test_01155_ordinary.dst TO test_01155_atomic.dst; RENAME TABLE test_01155_ordinary.src TO test_01155_atomic.src; -SET check_table_dependencies=0; +SET check_table_dependencies=0; -- Otherwise we'll get error "test_01155_atomic.dict depends on test_01155_ordinary.dist" in the next line. RENAME TABLE test_01155_ordinary.dist TO test_01155_atomic.dist; SET check_table_dependencies=1; RENAME DICTIONARY test_01155_ordinary.dict TO test_01155_atomic.dict; @@ -65,7 +65,9 @@ SELECT dictGet('test_01155_ordinary.dict', 'x', 'after renaming database'); SELECT database, substr(name, 1, 10) FROM system.tables WHERE database like 'test_01155_%'; -- Move tables back +SET check_table_dependencies=0; -- Otherwise we'll get error "test_01155_atomic.dict depends on test_01155_ordinary.dist" in the next line. RENAME DATABASE test_01155_ordinary TO test_01155_atomic; +SET check_table_dependencies=1; set allow_deprecated_database_ordinary=1; CREATE DATABASE test_01155_ordinary ENGINE=Ordinary; diff --git a/tests/queries/0_stateless/01167_isolation_hermitage.sh b/tests/queries/0_stateless/01167_isolation_hermitage.sh index 3f2c8308216..1d1e8006d1d 100755 --- a/tests/queries/0_stateless/01167_isolation_hermitage.sh +++ b/tests/queries/0_stateless/01167_isolation_hermitage.sh @@ -8,24 +8,37 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh # shellcheck source=./transactions.lib . "$CURDIR"/transactions.lib +# shellcheck source=./parts.lib +. "$CURDIR"/parts.lib set -e # https://github.com/ept/hermitage -$CLICKHOUSE_CLIENT -q "drop table if exists test" -$CLICKHOUSE_CLIENT -q "create table test (id int, value int) engine=MergeTree order by id" +function hard_reset_table() +{ + # Merges aren;t blocked, when they runs they left parts which are removed after old_parts_lifetime + # Test have to set old_parts_lifetime in low value in order to be able to wait deleting empty parts + $CLICKHOUSE_CLIENT -q "drop table if exists test" + $CLICKHOUSE_CLIENT -q "create table test (id int, value int) engine=MergeTree order by id SETTINGS old_parts_lifetime = 5" + $CLICKHOUSE_CLIENT -q "insert into test (id, value) values (1, 10);" + $CLICKHOUSE_CLIENT -q "insert into test (id, value) values (2, 20);" +} function reset_table() { $CLICKHOUSE_CLIENT -q "truncate table test;" $CLICKHOUSE_CLIENT -q "insert into test (id, value) values (1, 10);" $CLICKHOUSE_CLIENT -q "insert into test (id, value) values (2, 20);" + + # The is a chance that old parts are held by the oldest snapshot existed on a node + # In order not to wait too long (>60s) there is used a fallback to table recreation + wait_for_delete_empty_parts "test" $CLICKHOUSE_DATABASE 1>/dev/null 2>&1 || hard_reset_table } # TODO update test after implementing Read Committed # G0 -reset_table +hard_reset_table tx 1 "begin transaction" tx 2 "begin transaction" tx 1 "alter table test update value=11 where id=1" @@ -109,6 +122,7 @@ tx_wait 12 tx_wait 13 $CLICKHOUSE_CLIENT -q "select 16, * from test order by id" + # PMP write reset_table tx 14 "begin transaction" diff --git a/tests/queries/0_stateless/01168_mutations_isolation.reference b/tests/queries/0_stateless/01168_mutations_isolation.reference index 1b3e3f145b1..f9ebd1c5f83 100644 --- a/tests/queries/0_stateless/01168_mutations_isolation.reference +++ b/tests/queries/0_stateless/01168_mutations_isolation.reference @@ -21,18 +21,18 @@ tx7 7 20 all_1_1_0_13 tx7 7 40 all_14_14_0 tx7 7 60 all_7_7_0_13 tx7 7 80 all_12_12_0_13 -tx7 8 20 all_1_14_1_13 -tx7 8 40 all_1_14_1_13 -tx7 8 60 all_1_14_1_13 -tx7 8 80 all_1_14_1_13 +tx7 8 20 all_1_14_2_13 +tx7 8 40 all_1_14_2_13 +tx7 8 60 all_1_14_2_13 +tx7 8 80 all_1_14_2_13 Serialization error INVALID_TRANSACTION -tx11 9 21 all_1_14_1_17 -tx11 9 41 all_1_14_1_17 -tx11 9 61 all_1_14_1_17 -tx11 9 81 all_1_14_1_17 +tx11 9 21 all_1_14_2_17 +tx11 9 41 all_1_14_2_17 +tx11 9 61 all_1_14_2_17 +tx11 9 81 all_1_14_2_17 1 1 RUNNING -tx14 10 22 all_1_14_1_18 -tx14 10 42 all_1_14_1_18 -tx14 10 62 all_1_14_1_18 -tx14 10 82 all_1_14_1_18 +tx14 10 22 all_1_14_2_18 +tx14 10 42 all_1_14_2_18 +tx14 10 62 all_1_14_2_18 +tx14 10 82 all_1_14_2_18 diff --git a/tests/queries/0_stateless/01168_mutations_isolation.sh b/tests/queries/0_stateless/01168_mutations_isolation.sh index ebfdffdaeee..5d014e030f1 100755 --- a/tests/queries/0_stateless/01168_mutations_isolation.sh +++ b/tests/queries/0_stateless/01168_mutations_isolation.sh @@ -53,6 +53,9 @@ tx 6 "alter table mt update n=n*10 wh tx 6 "insert into mt values (40)" tx 6 "commit" +function accept_both_parts() { + sed 's/all_1_14_1_1/all_1_14_2_1/g' +} tx 7 "begin transaction" tx 7 "select 7, n, _part from mt order by n" @@ -61,7 +64,7 @@ tx_async 8 "alter table mt update n = 0 whe $CLICKHOUSE_CLIENT -q "kill mutation where database=currentDatabase() and mutation_id='mutation_15.txt' format Null" 2>&1| grep -Fv "probably it finished" tx_sync 8 "rollback" tx 7 "optimize table mt final" -tx 7 "select 8, n, _part from mt order by n" +tx 7 "select 8, n, _part from mt order by n" | accept_both_parts tx 10 "begin transaction" tx 10 "alter table mt update n = 0 where 1" | grep -Eo "Serialization error" | uniq tx 7 "alter table mt update n=n+1 where 1" @@ -71,7 +74,7 @@ tx 7 "commit" tx_async 11 "begin transaction" -tx_async 11 "select 9, n, _part from mt order by n" +tx_async 11 "select 9, n, _part from mt order by n" | accept_both_parts tx_async 12 "begin transaction" tx_async 11 "alter table mt update n=n+1 where 1" >/dev/null tx_async 12 "alter table mt update n=n+1 where 1" >/dev/null @@ -88,6 +91,6 @@ $CLICKHOUSE_CLIENT -q "kill transaction where tid=$tid_to_kill format Null" tx_sync 13 "rollback" tx 14 "begin transaction" -tx 14 "select 10, n, _part from mt order by n" +tx 14 "select 10, n, _part from mt order by n" | accept_both_parts $CLICKHOUSE_CLIENT --database_atomic_wait_for_drop_and_detach_synchronously=0 -q "drop table mt" diff --git a/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh b/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh index 32ad78dead6..3fb3730f758 100755 --- a/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh +++ b/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh @@ -6,8 +6,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./transactions.lib +. "$CURDIR"/transactions.lib -set -e +set -eu $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS src"; $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS dst"; @@ -16,7 +18,7 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE dst (n UInt64, type UInt8) ENGINE=Merge function thread_insert() { - set -e + set -eu val=1 while true; do $CLICKHOUSE_CLIENT --multiquery --query " @@ -29,64 +31,184 @@ function thread_insert() done } +function is_tx_aborted_with() +{ + grep_args="" + for pattern in "${@}"; do + grep_args="$grep_args -Fe $pattern" + done + + grep $grep_args >/dev/null +} + +function is_tx_failed() +{ + grep -Fe 'DB::Exception:' > /dev/null +} + +function is_tx_ok() +{ + is_tx_failed && return 1 +} # NOTE # ALTER PARTITION query stops merges, -# but serialization error is still possible if some merge was assigned (and committed) between BEGIN and ALTER. +# but parts could be deleted (SERIALIZATION_ERROR) if some merge was assigned (and committed) between BEGIN and ALTER. function thread_partition_src_to_dst() { - set -e + set -eu count=0 sum=0 for i in {1..20}; do - out=$( - $CLICKHOUSE_CLIENT --multiquery --query " - BEGIN TRANSACTION; - INSERT INTO src VALUES /* ($i, 3) */ ($i, 3); - INSERT INTO dst SELECT * FROM src; - ALTER TABLE src DROP PARTITION ID 'all'; - SET throw_on_unsupported_query_inside_transaction=0; - SELECT throwIf((SELECT (count(), sum(n)) FROM merge(currentDatabase(), '') WHERE type=3) != ($count + 1, $sum + $i)) FORMAT Null; - COMMIT;" 2>&1) ||: + session_id="_src_to_dst_$i" + session_id_debug="_src_to_dst_debug_$i" + + tx $session_id "BEGIN TRANSACTION" + tx_id=$(tx $session_id "select transactionID().1" | awk '{print $2}') + + tx $session_id "INSERT INTO src VALUES /* ($i, 3) */ ($i, 3)" + tx $session_id "INSERT INTO dst SELECT * FROM src" + + output=$(tx $session_id "ALTER TABLE src DROP PARTITION ID 'all'" ||:) + if echo "$output" | is_tx_aborted_with "SERIALIZATION_ERROR" "PART_IS_TEMPORARILY_LOCKED" "PART_IS_TEMPORARILY_LOCKED" + then + tx $session_id "ROLLBACK" + continue + fi + + if echo "$output" | is_tx_failed + then + echo "thread_partition_src_to_dst tx_id: $tx_id session_id: $session_id" >&2 + echo "drop part has failed with unexpected status" >&2 + echo -e "output:\n $output" >&2 + return 1 + fi + + tx $session_id "SET throw_on_unsupported_query_inside_transaction=0" + + trace_output="" + output=$(tx $session_id "select transactionID()") + trace_output="$trace_output $output\n" + + tx $session_id_debug "begin transaction" + tx $session_id_debug "set transaction snapshot 3" + output=$(tx $session_id_debug "select 'src_to_dst', $i, 'src', type, n, _part from src order by type, n") + trace_output="$trace_output $output\n" + output=$(tx $session_id_debug "select 'src_to_dst', $i, 'dst', type, n, _part from dst order by type, n") + trace_output="$trace_output $output\n" + tx $session_id_debug "commit" + + output=$(tx $session_id "SELECT throwIf((SELECT (count(), sum(n)) FROM merge(currentDatabase(), '') WHERE type=3) != ($count + 1, $sum + $i)) FORMAT Null" ||:) + if echo "$output" | is_tx_aborted_with "FUNCTION_THROW_IF_VALUE_IS_NON_ZERO" + then + echo "thread_partition_src_to_dst tx_id: $tx_id session_id: $session_id" >&2 + echо "select throwIf has failed with FUNCTION_THROW_IF_VALUE_IS_NON_ZERO" >&2 + echo -e "trace_output:\n $trace_output" >&2 + echo -e "output:\n $output" >&2 + return 1 + fi + + if echo "$output" | is_tx_failed + then + echo "thread_partition_src_to_dst tx_id: $tx_id session_id: $session_id" >&2 + echo "select throwIf has failed with unexpected status" >&2 + echo -e "trace_output:\n $trace_output" >&2 + echo -e "output:\n $output" >&2 + return 1 + fi + + tx $session_id "COMMIT" + + count=$((count + 1)) + sum=$((sum + i)) - echo "$out" | grep -Fv "SERIALIZATION_ERROR" | grep -F "Received from " && $CLICKHOUSE_CLIENT --multiquery --query " - begin transaction; - set transaction snapshot 3; - select $i, 'src', type, n, _part from src order by type, n; - select $i, 'dst', type, n, _part from dst order by type, n; - rollback" ||: - echo "$out" | grep -Fa "SERIALIZATION_ERROR" >/dev/null || count=$((count+1)) - echo "$out" | grep -Fa "SERIALIZATION_ERROR" >/dev/null || sum=$((sum+i)) done } function thread_partition_dst_to_src() { - set -e - for i in {1..20}; do + set -eu + i=0 + while (( i <= 20 )); do + session_id="_dst_to_src_$i" + session_id_debug="_dst_to_src_debug_$i" + + tx $session_id "SYSTEM STOP MERGES dst" + tx $session_id "ALTER TABLE dst DROP PARTITION ID 'nonexistent';" + tx $session_id "SYSTEM SYNC TRANSACTION LOG" + + tx $session_id "BEGIN TRANSACTION" + tx_id=$(tx $session_id "select transactionID().1" | awk '{print $2}') + + tx $session_id "INSERT INTO dst VALUES /* ($i, 4) */ ($i, 4)" + tx $session_id "INSERT INTO src SELECT * FROM dst" + + output=$(tx $session_id "ALTER TABLE dst DROP PARTITION ID 'all'" ||:) + if echo "$output" | is_tx_aborted_with "PART_IS_TEMPORARILY_LOCKED" + then + # this is legit case, just retry + tx $session_id "ROLLBACK" + continue + fi + + if echo "$output" | is_tx_failed + then + echo "thread_partition_dst_to_src tx_id: $tx_id session_id: $session_id" >&2 + echo "drop part has failed with unexpected status" >&2 + echo "output $output" >&2 + return 1 + fi + + tx $session_id "SET throw_on_unsupported_query_inside_transaction=0" + tx $session_id "SYSTEM START MERGES dst" + + trace_output="" + output=$(tx $session_id "select transactionID()") + trace_output="$trace_output $output" + + tx $session_id_debug "begin transaction" + tx $session_id_debug "set transaction snapshot 3" + output=$(tx $session_id_debug "select 'dst_to_src', $i, 'src', type, n, _part from src order by type, n") + trace_output="$trace_output $output" + output=$(tx $session_id_debug "select 'dst_to_src', $i, 'dst', type, n, _part from dst order by type, n") + trace_output="$trace_output $output" + tx $session_id_debug "commit" + + output=$(tx $session_id "SELECT throwIf((SELECT (count(), sum(n)) FROM merge(currentDatabase(), '') WHERE type=4) != (toUInt8($i/2 + 1), (select sum(number) from numbers(1, $i) where number % 2 or number=$i))) FORMAT Null" ||:) + if echo "$output" | is_tx_aborted_with "FUNCTION_THROW_IF_VALUE_IS_NON_ZERO" + then + echo "thread_partition_dst_to_src tx_id: $tx_id session_id: $session_id" >&2 + echo "select throwIf has failed with FUNCTION_THROW_IF_VALUE_IS_NON_ZERO" >&2 + echo -e "trace_output:\n $trace_output" >&2 + echo -e "output:\n $output" >&2 + return 1 + fi + + if echo "$output" | is_tx_failed + then + echo "thread_partition_dst_to_src tx_id: $tx_id session_id: $session_id" >&2 + echo "SELECT throwIf has failed with unexpected status" >&2 + echo -e "trace_output:\n $trace_output" >&2 + echo -e "output:\n $output" >&2 + return 1 + fi + action="ROLLBACK" if (( i % 2 )); then action="COMMIT" fi - $CLICKHOUSE_CLIENT --multiquery --query " - SYSTEM STOP MERGES dst; - ALTER TABLE dst DROP PARTITION ID 'nonexistent'; -- STOP MERGES doesn't wait for started merges to finish, so we use this trick - SYSTEM SYNC TRANSACTION LOG; - BEGIN TRANSACTION; - INSERT INTO dst VALUES /* ($i, 4) */ ($i, 4); - INSERT INTO src SELECT * FROM dst; - ALTER TABLE dst DROP PARTITION ID 'all'; - SET throw_on_unsupported_query_inside_transaction=0; - SYSTEM START MERGES dst; - SELECT throwIf((SELECT (count(), sum(n)) FROM merge(currentDatabase(), '') WHERE type=4) != (toUInt8($i/2 + 1), (select sum(number) from numbers(1, $i) where number % 2 or number=$i))) FORMAT Null; - $action;" + + tx $session_id "$action" + + i=$((i + 1)) done } function thread_select() { - set -e + set -eu while true; do + output=$( $CLICKHOUSE_CLIENT --multiquery --query " BEGIN TRANSACTION; -- no duplicates @@ -94,10 +216,14 @@ function thread_select() SELECT type, throwIf(count(n) != countDistinct(n)) FROM dst GROUP BY type FORMAT Null; -- rows inserted by thread_insert moved together SET throw_on_unsupported_query_inside_transaction=0; + SELECT _table, throwIf(arraySort(groupArrayIf(n, type=1)) != arraySort(groupArrayIf(n, type=2))) FROM merge(currentDatabase(), '') GROUP BY _table FORMAT Null; + -- all rows are inserted in insert_thread SELECT type, throwIf(count(n) != max(n)), throwIf(sum(n) != max(n)*(max(n)+1)/2) FROM merge(currentDatabase(), '') WHERE type IN (1, 2) GROUP BY type ORDER BY type FORMAT Null; - COMMIT;" + COMMIT;" 2>&1 ||:) + + echo "$output" | grep -F "Received from " > /dev/null && echo "$output">&2 && return 1 done } @@ -106,11 +232,13 @@ thread_select & PID_2=$! thread_partition_src_to_dst & PID_3=$! thread_partition_dst_to_src & PID_4=$! -wait $PID_3 && wait $PID_4 +wait $PID_3 +wait $PID_4 kill -TERM $PID_1 kill -TERM $PID_2 -wait +wait ||: + wait_for_queries_to_finish $CLICKHOUSE_CLIENT -q "SELECT type, count(n) = countDistinct(n) FROM merge(currentDatabase(), '') GROUP BY type ORDER BY type" @@ -118,6 +246,5 @@ $CLICKHOUSE_CLIENT -q "SELECT DISTINCT arraySort(groupArrayIf(n, type=1)) = arra $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM merge(currentDatabase(), '') WHERE type=4" $CLICKHOUSE_CLIENT -q "SELECT type, count(n) == max(n), sum(n) == max(n)*(max(n)+1)/2 FROM merge(currentDatabase(), '') WHERE type IN (1, 2) GROUP BY type ORDER BY type" - $CLICKHOUSE_CLIENT --query "DROP TABLE src"; $CLICKHOUSE_CLIENT --query "DROP TABLE dst"; diff --git a/tests/queries/0_stateless/01172_transaction_counters.reference b/tests/queries/0_stateless/01172_transaction_counters.reference index 3a167e76817..3099fae4a42 100644 --- a/tests/queries/0_stateless/01172_transaction_counters.reference +++ b/tests/queries/0_stateless/01172_transaction_counters.reference @@ -28,9 +28,13 @@ 4 1 Commit 1 1 1 0 5 1 Begin 1 1 1 1 5 1 AddPart 1 1 1 1 all_5_5_0 +5 1 AddPart 1 1 1 1 all_1_1_1 5 1 LockPart 1 1 1 1 all_1_1_0 +5 1 AddPart 1 1 1 1 all_3_3_1 5 1 LockPart 1 1 1 1 all_3_3_0 +5 1 AddPart 1 1 1 1 all_4_4_1 5 1 LockPart 1 1 1 1 all_4_4_0 +5 1 AddPart 1 1 1 1 all_5_5_1 5 1 LockPart 1 1 1 1 all_5_5_0 5 1 UnlockPart 1 1 1 1 all_1_1_0 5 1 UnlockPart 1 1 1 1 all_3_3_0 diff --git a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh index c18514d0ecc..d2695e602c5 100755 --- a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh +++ b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh @@ -22,7 +22,7 @@ function run_until_out_contains() PATTERN=$1 shift - for ((i=MIN_TIMEOUT; i<10; i++)) + for ((i=MIN_TIMEOUT; i<33; i=i*2)) do "$@" --distributed_ddl_task_timeout="$i" > "$TMP_OUT" 2>&1 if grep -q "$PATTERN" "$TMP_OUT" @@ -37,7 +37,7 @@ RAND_COMMENT="01175_DDL_$RANDOM" LOG_COMMENT="${CLICKHOUSE_LOG_COMMENT}_$RAND_COMMENT" CLICKHOUSE_CLIENT_WITH_SETTINGS=${CLICKHOUSE_CLIENT/--log_comment ${CLICKHOUSE_LOG_COMMENT}/--log_comment ${LOG_COMMENT}} -CLICKHOUSE_CLIENT_WITH_SETTINGS+=" --output_format_parallel_formatting=0 " +CLICKHOUSE_CLIENT_WITH_SETTINGS+=" --output_format_parallel_formatting=0 --database_atomic_wait_for_drop_and_detach_synchronously=0 " CLIENT=${CLICKHOUSE_CLIENT_WITH_SETTINGS} CLIENT+=" --distributed_ddl_task_timeout=$TIMEOUT " diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index dd9fa7abc1b..f2c3e8eda9d 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -39,6 +39,7 @@ ALTER MOVE PARTITION ['ALTER MOVE PART','MOVE PARTITION','MOVE PART'] TABLE ALTE ALTER FETCH PARTITION ['ALTER FETCH PART','FETCH PARTITION'] TABLE ALTER TABLE ALTER FREEZE PARTITION ['FREEZE PARTITION','UNFREEZE'] TABLE ALTER TABLE ALTER DATABASE SETTINGS ['ALTER DATABASE SETTING','ALTER MODIFY DATABASE SETTING','MODIFY DATABASE SETTING'] DATABASE ALTER DATABASE +ALTER NAMED COLLECTION [] \N ALTER ALTER TABLE [] \N ALTER ALTER DATABASE [] \N ALTER ALTER VIEW REFRESH ['ALTER LIVE VIEW REFRESH','REFRESH VIEW'] VIEW ALTER VIEW @@ -51,12 +52,14 @@ CREATE VIEW [] VIEW CREATE CREATE DICTIONARY [] DICTIONARY CREATE CREATE TEMPORARY TABLE [] GLOBAL CREATE CREATE FUNCTION [] GLOBAL CREATE +CREATE NAMED COLLECTION [] GLOBAL CREATE CREATE [] \N ALL DROP DATABASE [] DATABASE DROP DROP TABLE [] TABLE DROP DROP VIEW [] VIEW DROP DROP DICTIONARY [] DICTIONARY DROP DROP FUNCTION [] GLOBAL DROP +DROP NAMED COLLECTION [] GLOBAL DROP DROP [] \N ALL TRUNCATE ['TRUNCATE TABLE'] TABLE ALL OPTIMIZE ['OPTIMIZE TABLE'] TABLE ALL diff --git a/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql index 2814c87c933..f57ebc10da2 100644 --- a/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql +++ b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS data_01283; +set allow_asynchronous_read_from_io_pool_for_merge_tree = 0; set remote_filesystem_read_method = 'read'; set local_filesystem_read_method = 'pread'; set load_marks_asynchronously = 0; diff --git a/tests/queries/0_stateless/01323_too_many_threads_bug.sql b/tests/queries/0_stateless/01323_too_many_threads_bug.sql index c2cce81d200..c377e2c7570 100644 --- a/tests/queries/0_stateless/01323_too_many_threads_bug.sql +++ b/tests/queries/0_stateless/01323_too_many_threads_bug.sql @@ -3,6 +3,7 @@ drop table if exists table_01323_many_parts; set remote_filesystem_read_method = 'read'; set local_filesystem_read_method = 'pread'; set load_marks_asynchronously = 0; +set allow_asynchronous_read_from_io_pool_for_merge_tree = 0; create table table_01323_many_parts (x UInt64) engine = MergeTree order by x partition by x % 100; set max_partitions_per_insert_block = 100; diff --git a/tests/queries/0_stateless/01451_detach_drop_part.reference b/tests/queries/0_stateless/01451_detach_drop_part.reference index bc4f1b6be80..a34c308cb72 100644 --- a/tests/queries/0_stateless/01451_detach_drop_part.reference +++ b/tests/queries/0_stateless/01451_detach_drop_part.reference @@ -10,6 +10,8 @@ all_2_2_0 -- drop part -- 0 2 +all_1_1_0 +all_3_3_0 -- resume merges -- 0 2 diff --git a/tests/queries/0_stateless/01451_detach_drop_part.sql b/tests/queries/0_stateless/01451_detach_drop_part.sql index a285730e45f..4c6cf54a6d9 100644 --- a/tests/queries/0_stateless/01451_detach_drop_part.sql +++ b/tests/queries/0_stateless/01451_detach_drop_part.sql @@ -31,6 +31,8 @@ ALTER TABLE mt_01451 ATTACH PART 'all_4_4_0'; -- { serverError 233 } SELECT v FROM mt_01451 ORDER BY v; +SELECT name FROM system.parts WHERE table = 'mt_01451' AND active AND database = currentDatabase(); + SELECT '-- resume merges --'; SYSTEM START MERGES mt_01451; OPTIMIZE TABLE mt_01451 FINAL; diff --git a/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql b/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql index dafe652d271..e3bc8cf6e72 100644 --- a/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql +++ b/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS select_final; +SET allow_asynchronous_read_from_io_pool_for_merge_tree = 0; SET do_not_merge_across_partitions_select_final = 1; SET max_threads = 16; diff --git a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order_long.sh b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order_long.sh index fc79725aebe..bfb4601e62b 100755 --- a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order_long.sh +++ b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order_long.sh @@ -19,16 +19,16 @@ $CLICKHOUSE_CLIENT -q "create table ${name}_n_x engine=MergeTree order by (n, x) $CLICKHOUSE_CLIENT -q "optimize table ${name}_n final" $CLICKHOUSE_CLIENT -q "optimize table ${name}_n_x final" -$CLICKHOUSE_CLIENT -q "select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=0, optimize_read_in_window_order=0, max_memory_usage=$max_memory_usage, max_threads=1 format Null" 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo 'OK' || echo 'FAIL' +$CLICKHOUSE_CLIENT --allow_asynchronous_read_from_io_pool_for_merge_tree=0 -q "select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=0, optimize_read_in_window_order=0, max_memory_usage=$max_memory_usage, max_threads=1 format Null" 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo 'OK' || echo 'FAIL' $CLICKHOUSE_CLIENT -q "select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=1, max_memory_usage=$max_memory_usage, max_threads=1 format Null" -$CLICKHOUSE_CLIENT -q "select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0, optimize_read_in_window_order=0, max_memory_usage=$max_memory_usage, max_threads=1 format Null" 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo 'OK' || echo 'FAIL' +$CLICKHOUSE_CLIENT --allow_asynchronous_read_from_io_pool_for_merge_tree=0 -q "select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0, optimize_read_in_window_order=0, max_memory_usage=$max_memory_usage, max_threads=1 format Null" 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo 'OK' || echo 'FAIL' $CLICKHOUSE_CLIENT -q "select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1, max_memory_usage=$max_memory_usage, max_threads=1 format Null" -$CLICKHOUSE_CLIENT -q "select n, sum(x) OVER (PARTITION BY n ORDER BY x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0, optimize_read_in_window_order=0, max_memory_usage=$max_memory_usage, max_threads=1 format Null" 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo 'OK' || echo 'FAIL' +$CLICKHOUSE_CLIENT --allow_asynchronous_read_from_io_pool_for_merge_tree=0 -q "select n, sum(x) OVER (PARTITION BY n ORDER BY x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0, optimize_read_in_window_order=0, max_memory_usage=$max_memory_usage, max_threads=1 format Null" 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo 'OK' || echo 'FAIL' $CLICKHOUSE_CLIENT -q "select n, sum(x) OVER (PARTITION BY n ORDER BY x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1, max_memory_usage=$max_memory_usage, max_threads=1 format Null" -$CLICKHOUSE_CLIENT -q "select n, sum(x) OVER (PARTITION BY n+x%2 ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1, max_memory_usage=$max_memory_usage, max_threads=1 format Null" 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo 'OK' || echo 'FAIL' +$CLICKHOUSE_CLIENT --allow_asynchronous_read_from_io_pool_for_merge_tree=0 -q "select n, sum(x) OVER (PARTITION BY n+x%2 ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1, max_memory_usage=$max_memory_usage, max_threads=1 format Null" 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo 'OK' || echo 'FAIL' $CLICKHOUSE_CLIENT -q "drop table ${name}" $CLICKHOUSE_CLIENT -q "drop table ${name}_n" diff --git a/tests/queries/0_stateless/01660_system_parts_smoke.reference b/tests/queries/0_stateless/01660_system_parts_smoke.reference index 36550f31bd0..b38d699c2b9 100644 --- a/tests/queries/0_stateless/01660_system_parts_smoke.reference +++ b/tests/queries/0_stateless/01660_system_parts_smoke.reference @@ -9,6 +9,6 @@ all_2_2_0 1 1 Active 2 Outdated # truncate -Outdated -Outdated +HAVE PARTS Active +HAVE PARTS Outdated # drop diff --git a/tests/queries/0_stateless/01660_system_parts_smoke.sql b/tests/queries/0_stateless/01660_system_parts_smoke.sql index cc925680425..64cba86b8f6 100644 --- a/tests/queries/0_stateless/01660_system_parts_smoke.sql +++ b/tests/queries/0_stateless/01660_system_parts_smoke.sql @@ -31,9 +31,11 @@ OPTIMIZE TABLE data_01660 FINAL; SELECT count(), _state FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660' GROUP BY _state ORDER BY _state; -- TRUNCATE does not remove parts instantly +-- Empty active parts are clearing by async process +-- Inactive parts are clearing by async process also SELECT '# truncate'; TRUNCATE data_01660; -SELECT _state FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660'; +SELECT if (count() > 0, 'HAVE PARTS', 'NO PARTS'), _state FROM system.parts WHERE database = currentDatabase() AND table = 'data_01660' GROUP BY _state; -- But DROP does SELECT '# drop'; diff --git a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.reference b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.reference new file mode 100644 index 00000000000..c66682ca038 --- /dev/null +++ b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.reference @@ -0,0 +1,21 @@ +# clickhouse-client +concatAssumeInjective: OK +ReplacingMergeTree: OK +JSONEachRow: OK +clusterAllReplicas: OK +SimpleAggregateFunction: OK +write_ahead_log_interval_ms_to_fsync: OK +max_concurrent_queries_for_all_users: OK +test_shard_localhost: OK +default_path_test: OK +default: OK +uniqCombined64ForEach: OK +system: OK +aggregate_function_combinators: OK +primary_key_bytes_in_memory_allocated: OK +# clickhouse-local +concatAssumeInjective: OK +ReplacingMergeTree: OK +JSONEachRow: OK +clusterAllReplicas: OK +SimpleAggregateFunction: OK diff --git a/tests/queries/0_stateless/01676_long_clickhouse_client_autocomplete.sh b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh similarity index 64% rename from tests/queries/0_stateless/01676_long_clickhouse_client_autocomplete.sh rename to tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh index 1be082a6aae..056613c11b5 100755 --- a/tests/queries/0_stateless/01676_long_clickhouse_client_autocomplete.sh +++ b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh @@ -5,9 +5,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +SCRIPT_PATH="$CURDIR/$CLICKHOUSE_TEST_UNIQUE_NAME.generated-expect" + # NOTE: database = $CLICKHOUSE_DATABASE is superfluous -function test_completion_word_client() +function test_completion_word() { local w=$1 && shift @@ -15,10 +17,20 @@ function test_completion_word_client() local compword_begin=${w:0:$((w_len-3))} local compword_end=${w:$((w_len-3))} - # NOTE: here and below you should escape variables of the expect. - timeout 60s expect << EOF + # NOTE: + # - here and below you should escape variables of the expect. + # - you should not use "expect <<..." since in this case timeout/eof will + # not work (I guess due to attached stdin) + cat > "$SCRIPT_PATH" << EOF +# NOTE: log will be appended +exp_internal -f $CLICKHOUSE_TMP/$(basename "${BASH_SOURCE[0]}").debuglog 0 + +# NOTE: when expect have EOF on stdin it also closes stdout, so let's reopen it +# again for logging +set stdout_channel [open "/dev/stdout" w] + log_user 0 -set timeout 3 +set timeout 60 match_max 100000 expect_after { # Do not ignore eof from expect @@ -27,7 +39,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "$CLICKHOUSE_CLIENT_BINARY $CLICKHOUSE_CLIENT_OPT" +spawn bash -c "$*" expect ":) " # Make a query @@ -36,10 +48,12 @@ expect "SET $compword_begin" # Wait for suggestions to load, they are loaded in background set is_done 0 +set timeout 1 while {\$is_done == 0} { send -- "\\t" expect { "$compword_begin$compword_end" { + puts \$stdout_channel "$compword_begin$compword_end: OK" set is_done 1 } default { @@ -48,9 +62,18 @@ while {\$is_done == 0} { } } +close \$stdout_channel + send -- "\\3\\4" expect eof EOF + + # NOTE: run expect under timeout since there is while loop that is not + # limited with timeout. + # + # NOTE: cat is required to serialize stdout for expect (without this pipe + # it will reopen the file again, and the output will be mixed). + timeout 2m expect -f "$SCRIPT_PATH" | cat } # last 3 bytes will be completed, @@ -90,53 +113,6 @@ client_compwords_positive=( # FIXME: none ) - -function test_completion_word_local() -{ - local w=$1 && shift - - local w_len=${#w} - local compword_begin=${w:0:$((w_len-3))} - local compword_end=${w:$((w_len-3))} - - # NOTE: here and below you should escape variables of the expect. - timeout 60s expect << EOF -log_user 0 -set timeout 3 -match_max 100000 -expect_after { - # Do not ignore eof from expect - eof { exp_continue } - # A default timeout action is to do nothing, change it to fail - timeout { exit 1 } -} - -spawn bash -c "$CLICKHOUSE_LOCAL" -expect ":) " - -# Make a query -send -- "SET $compword_begin" -expect "SET $compword_begin" - -# Wait for suggestions to load, they are loaded in background -set is_done 0 -while {\$is_done == 0} { - send -- "\\t" - expect { - "$compword_begin$compword_end" { - set is_done 1 - } - default { - sleep 1 - } - } -} - -send -- "\\3\\4" -expect eof -EOF -} - local_compwords_positive=( # system.functions concatAssumeInjective @@ -150,12 +126,15 @@ local_compwords_positive=( SimpleAggregateFunction ) +echo "# clickhouse-client" for w in "${client_compwords_positive[@]}"; do - test_completion_word_client "$w" || echo "[FAIL] $w (positive)" + test_completion_word "$w" "$CLICKHOUSE_CLIENT" +done +echo "# clickhouse-local" +for w in "${local_compwords_positive[@]}"; do + test_completion_word "$w" "$CLICKHOUSE_LOCAL" done -for w in "${local_compwords_positive[@]}"; do - test_completion_word_local "$w" || echo "[FAIL] $w (positive)" -done +rm -f "${SCRIPT_PATH:?}" exit 0 diff --git a/tests/queries/0_stateless/01686_event_time_microseconds_part_log.sh b/tests/queries/0_stateless/01686_event_time_microseconds_part_log.sh new file mode 100755 index 00000000000..db53dbbce85 --- /dev/null +++ b/tests/queries/0_stateless/01686_event_time_microseconds_part_log.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q 'DROP TABLE IF EXISTS table_with_single_pk' + +${CLICKHOUSE_CLIENT} -q ' + CREATE TABLE table_with_single_pk + ( + key UInt8, + value String + ) + ENGINE = MergeTree + ORDER BY key +' + +${CLICKHOUSE_CLIENT} -q 'INSERT INTO table_with_single_pk SELECT number, toString(number % 10) FROM numbers(1000000)' + +# Check NewPart +${CLICKHOUSE_CLIENT} -q 'SYSTEM FLUSH LOGS' +${CLICKHOUSE_CLIENT} -q " + WITH ( + SELECT (event_time, event_time_microseconds) + FROM system.part_log + WHERE table = 'table_with_single_pk' AND database = currentDatabase() AND event_type = 'NewPart' + ORDER BY event_time DESC + LIMIT 1 + ) AS time + SELECT if(dateDiff('second', toDateTime(time.2), toDateTime(time.1)) = 0, 'ok', 'fail')" + +# Now let's check RemovePart +${CLICKHOUSE_CLIENT} -q 'TRUNCATE TABLE table_with_single_pk' + +# Wait until parts are removed +function get_inactive_parts_count() { + table_name=$1 + ${CLICKHOUSE_CLIENT} -q " + SELECT + count() + FROM + system.parts + WHERE + table = 'table_with_single_pk' + AND + active = 0 + AND + database = '${CLICKHOUSE_DATABASE}' + " +} + +function wait_table_inactive_parts_are_gone() { + table_name=$1 + + while true + do + count=$(get_inactive_parts_count $table_name) + if [[ count -gt 0 ]] + then + sleep 1 + else + break + fi + done +} + +export -f get_inactive_parts_count +export -f wait_table_inactive_parts_are_gone +timeout 60 bash -c 'wait_table_inactive_parts_are_gone table_with_single_pk' + +${CLICKHOUSE_CLIENT} -q 'SYSTEM FLUSH LOGS;' +${CLICKHOUSE_CLIENT} -q " + WITH ( + SELECT (event_time, event_time_microseconds) + FROM system.part_log + WHERE table = 'table_with_single_pk' AND database = currentDatabase() AND event_type = 'RemovePart' + ORDER BY event_time DESC + LIMIT 1 + ) AS time + SELECT if(dateDiff('second', toDateTime(time.2), toDateTime(time.1)) = 0, 'ok', 'fail')" + +${CLICKHOUSE_CLIENT} -q 'DROP TABLE table_with_single_pk' + + diff --git a/tests/queries/0_stateless/01686_event_time_microseconds_part_log.sql b/tests/queries/0_stateless/01686_event_time_microseconds_part_log.sql deleted file mode 100644 index 6063be4d1da..00000000000 --- a/tests/queries/0_stateless/01686_event_time_microseconds_part_log.sql +++ /dev/null @@ -1,36 +0,0 @@ -DROP TABLE IF EXISTS table_with_single_pk; - -CREATE TABLE table_with_single_pk -( - key UInt8, - value String -) -ENGINE = MergeTree -ORDER BY key; - -INSERT INTO table_with_single_pk SELECT number, toString(number % 10) FROM numbers(1000000); - --- Check NewPart -SYSTEM FLUSH LOGS; -WITH ( - SELECT (event_time, event_time_microseconds) - FROM system.part_log - WHERE table = 'table_with_single_pk' AND database = currentDatabase() AND event_type = 'NewPart' - ORDER BY event_time DESC - LIMIT 1 - ) AS time -SELECT if(dateDiff('second', toDateTime(time.2), toDateTime(time.1)) = 0, 'ok', 'fail'); - --- Now let's check RemovePart -TRUNCATE TABLE table_with_single_pk; -SYSTEM FLUSH LOGS; -WITH ( - SELECT (event_time, event_time_microseconds) - FROM system.part_log - WHERE table = 'table_with_single_pk' AND database = currentDatabase() AND event_type = 'RemovePart' - ORDER BY event_time DESC - LIMIT 1 - ) AS time -SELECT if(dateDiff('second', toDateTime(time.2), toDateTime(time.1)) = 0, 'ok', 'fail'); - -DROP TABLE table_with_single_pk; diff --git a/tests/queries/0_stateless/01710_projection_detach_part.sql b/tests/queries/0_stateless/01710_projection_detach_part.sql index e3e6c7ac165..d28c0848d42 100644 --- a/tests/queries/0_stateless/01710_projection_detach_part.sql +++ b/tests/queries/0_stateless/01710_projection_detach_part.sql @@ -10,6 +10,6 @@ alter table t detach partition 1; alter table t attach partition 1; -select count() from system.projection_parts where database = currentDatabase() and table = 't'; +select count() from system.projection_parts where database = currentDatabase() and table = 't' and active; drop table t; diff --git a/tests/queries/0_stateless/01710_projection_with_joins.sql b/tests/queries/0_stateless/01710_projection_with_joins.sql index a54ba21fd27..5dac2f05da9 100644 --- a/tests/queries/0_stateless/01710_projection_with_joins.sql +++ b/tests/queries/0_stateless/01710_projection_with_joins.sql @@ -2,20 +2,20 @@ drop table if exists t; create table t (s UInt16, l UInt16, projection p (select s, l order by l)) engine MergeTree order by s; -select s from t join (select toUInt16(1) as s) x using (s) settings allow_experimental_projection_optimization = 1; -select s from t join (select toUInt16(1) as s) x using (s) settings allow_experimental_projection_optimization = 0; +select s from t join (select toUInt16(1) as s) x using (s) order by s settings allow_experimental_projection_optimization = 1; +select s from t join (select toUInt16(1) as s) x using (s) order by s settings allow_experimental_projection_optimization = 0; drop table t; drop table if exists mt; create table mt (id1 Int8, id2 Int8) Engine=MergeTree order by tuple(); -select id1 as alias1 from mt all inner join (select id2 as alias1 from mt) as t using (alias1) settings allow_experimental_projection_optimization = 1; -select id1 from mt all inner join (select id2 as id1 from mt) as t using (id1) settings allow_experimental_projection_optimization = 1; -select id2 as id1 from mt all inner join (select id1 from mt) as t using (id1) settings allow_experimental_projection_optimization = 1; +select id1 as alias1 from mt all inner join (select id2 as alias1 from mt) as t using (alias1) order by id1 settings allow_experimental_projection_optimization = 1; +select id1 from mt all inner join (select id2 as id1 from mt) as t using (id1) order by id1 settings allow_experimental_projection_optimization = 1; +select id2 as id1 from mt all inner join (select id1 from mt) as t using (id1) order by id1 settings allow_experimental_projection_optimization = 1; drop table mt; drop table if exists j; create table j (id1 Int8, id2 Int8, projection p (select id1, id2 order by id2)) Engine=MergeTree order by id1 settings index_granularity = 1; insert into j select number, number from numbers(10); -select id1 as alias1 from j all inner join (select id2 as alias1 from j where id2 in (1, 2, 3)) as t using (alias1) where id2 in (2, 3, 4) settings allow_experimental_projection_optimization = 1; +select id1 as alias1 from j all inner join (select id2 as alias1 from j where id2 in (1, 2, 3)) as t using (alias1) where id2 in (2, 3, 4) order by id1 settings allow_experimental_projection_optimization = 1; drop table j; diff --git a/tests/queries/0_stateless/01721_join_implicit_cast_long.reference b/tests/queries/0_stateless/01721_join_implicit_cast_long.reference deleted file mode 100644 index 07c240fa784..00000000000 --- a/tests/queries/0_stateless/01721_join_implicit_cast_long.reference +++ /dev/null @@ -1,1005 +0,0 @@ -=== hash === -= full = --4 0 196 --3 0 197 --2 0 198 --1 0 199 -0 0 200 -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -6 106 \N -7 107 \N -8 108 \N -9 109 \N -10 110 \N -= left = -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -6 106 \N -7 107 \N -8 108 \N -9 109 \N -10 110 \N -= right = --4 0 196 --3 0 197 --2 0 198 --1 0 199 -0 0 200 -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -= inner = -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -= full = -0 0 -4 -0 0 -3 -0 0 -2 -0 0 -1 -0 0 0 -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 0 -7 7 0 -8 8 0 -9 9 0 -10 10 0 -= left = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 0 -7 7 0 -8 8 0 -9 9 0 -10 10 0 -= right = -0 0 -4 -0 0 -3 -0 0 -2 -0 0 -1 -0 0 0 -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -= inner = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -= join on = -= full = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= left = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= right = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= inner = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= full = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= left = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= right = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= inner = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= agg = -1 -1 -1 -1 -1 -1 -0 -10 0 -1 55 1055 -0 0 -10 0 990 -1 55 15 1055 1015 -= types = -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -=== partial_merge === -= full = --4 0 196 --3 0 197 --2 0 198 --1 0 199 -0 0 200 -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -6 106 \N -7 107 \N -8 108 \N -9 109 \N -10 110 \N -= left = -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -6 106 \N -7 107 \N -8 108 \N -9 109 \N -10 110 \N -= right = --4 0 196 --3 0 197 --2 0 198 --1 0 199 -0 0 200 -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -= inner = -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -= full = -0 0 -4 -0 0 -3 -0 0 -2 -0 0 -1 -0 0 0 -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 0 -7 7 0 -8 8 0 -9 9 0 -10 10 0 -= left = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 0 -7 7 0 -8 8 0 -9 9 0 -10 10 0 -= right = -0 0 -4 -0 0 -3 -0 0 -2 -0 0 -1 -0 0 0 -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -= inner = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -= join on = -= full = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= left = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= right = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= inner = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= full = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= left = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= right = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= inner = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= agg = -1 -1 -1 -1 -1 -1 -0 -10 0 -1 55 1055 -0 0 -10 0 990 -1 55 15 1055 1015 -= types = -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -=== full_sorting_merge === -= full = --4 0 196 --3 0 197 --2 0 198 --1 0 199 -0 0 200 -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -6 106 \N -7 107 \N -8 108 \N -9 109 \N -10 110 \N -= left = -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -6 106 \N -7 107 \N -8 108 \N -9 109 \N -10 110 \N -= right = --4 0 196 --3 0 197 --2 0 198 --1 0 199 -0 0 200 -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -= inner = -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -= full = -0 0 -4 -0 0 -3 -0 0 -2 -0 0 -1 -0 0 0 -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 0 -7 7 0 -8 8 0 -9 9 0 -10 10 0 -= left = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 0 -7 7 0 -8 8 0 -9 9 0 -10 10 0 -= right = -0 0 -4 -0 0 -3 -0 0 -2 -0 0 -1 -0 0 0 -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -= inner = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -= join on = -= full = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= left = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= right = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= inner = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= full = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= left = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= right = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= inner = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= agg = -1 -1 -1 -1 -1 -1 -0 -10 0 -1 55 1055 -0 0 -10 0 990 -1 55 15 1055 1015 -= types = -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -=== auto === -= full = --4 0 196 --3 0 197 --2 0 198 --1 0 199 -0 0 200 -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -6 106 \N -7 107 \N -8 108 \N -9 109 \N -10 110 \N -= left = -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -6 106 \N -7 107 \N -8 108 \N -9 109 \N -10 110 \N -= right = --4 0 196 --3 0 197 --2 0 198 --1 0 199 -0 0 200 -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -= inner = -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -= full = -0 0 -4 -0 0 -3 -0 0 -2 -0 0 -1 -0 0 0 -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 0 -7 7 0 -8 8 0 -9 9 0 -10 10 0 -= left = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 0 -7 7 0 -8 8 0 -9 9 0 -10 10 0 -= right = -0 0 -4 -0 0 -3 -0 0 -2 -0 0 -1 -0 0 0 -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -= inner = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -= join on = -= full = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= left = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= right = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= inner = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= full = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= left = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 0 \N -7 107 0 \N -8 108 0 \N -9 109 0 \N -10 110 0 \N -= right = -0 0 -4 196 -0 0 -3 197 -0 0 -2 198 -0 0 -1 199 -0 0 0 200 -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= inner = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= agg = -1 -1 -1 -1 -1 -1 -0 -10 0 -1 55 1055 -0 0 -10 0 990 -1 55 15 1055 1015 -= types = -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -=== join use nulls === -= full = --4 \N 196 --3 \N 197 --2 \N 198 --1 \N 199 -0 \N 200 -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -6 106 \N -7 107 \N -8 108 \N -9 109 \N -10 110 \N -= left = -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -6 106 \N -7 107 \N -8 108 \N -9 109 \N -10 110 \N -= right = --4 \N 196 --3 \N 197 --2 \N 198 --1 \N 199 -0 \N 200 -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -= inner = -1 101 201 -2 102 202 -3 103 203 -4 104 204 -5 105 205 -= full = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 \N -7 7 \N -8 8 \N -9 9 \N -10 10 \N -\N \N -4 -\N \N -3 -\N \N -2 -\N \N -1 -\N \N 0 -= left = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 \N -7 7 \N -8 8 \N -9 9 \N -10 10 \N -= right = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -\N \N -4 -\N \N -3 -\N \N -2 -\N \N -1 -\N \N 0 -= inner = -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -= join on = -= full = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 \N \N -7 107 \N \N -8 108 \N \N -9 109 \N \N -10 110 \N \N -\N \N -4 196 -\N \N -3 197 -\N \N -2 198 -\N \N -1 199 -\N \N 0 200 -= left = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 \N \N -7 107 \N \N -8 108 \N \N -9 109 \N \N -10 110 \N \N -= right = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -\N \N -4 196 -\N \N -3 197 -\N \N -2 198 -\N \N -1 199 -\N \N 0 200 -= inner = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= full = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 \N \N -7 107 \N \N -8 108 \N \N -9 109 \N \N -10 110 \N \N -\N \N -4 196 -\N \N -3 197 -\N \N -2 198 -\N \N -1 199 -\N \N 0 200 -= left = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -6 106 \N \N -7 107 \N \N -8 108 \N \N -9 109 \N \N -10 110 \N \N -= right = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -\N \N -4 196 -\N \N -3 197 -\N \N -2 198 -\N \N -1 199 -\N \N 0 200 -= inner = -1 101 1 201 -2 102 2 202 -3 103 3 203 -4 104 4 204 -5 105 5 205 -= agg = -1 -1 -1 -1 -1 -1 -0 -10 \N -1 55 1055 -1 55 15 1055 1015 -\N \N -10 \N 990 -= types = -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 diff --git a/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2 b/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2 new file mode 100644 index 00000000000..e9f32087439 --- /dev/null +++ b/tests/queries/0_stateless/01721_join_implicit_cast_long.reference.j2 @@ -0,0 +1,446 @@ +{% for join_algorithm in ['hash', 'partial_merge', 'auto', 'full_sorting_merge', 'grace_hash'] -%} +=== {{ join_algorithm }} === += full = +{% if join_algorithm not in ['grace_hash'] -%} +-4 0 196 +-3 0 197 +-2 0 198 +-1 0 199 +0 0 200 +1 101 201 +2 102 202 +3 103 203 +4 104 204 +5 105 205 +6 106 \N +7 107 \N +8 108 \N +9 109 \N +10 110 \N +{% endif -%} += left = +1 101 201 +2 102 202 +3 103 203 +4 104 204 +5 105 205 +6 106 \N +7 107 \N +8 108 \N +9 109 \N +10 110 \N += right = +{% if join_algorithm not in ['grace_hash'] -%} +-4 0 196 +-3 0 197 +-2 0 198 +-1 0 199 +0 0 200 +1 101 201 +2 102 202 +3 103 203 +4 104 204 +5 105 205 +{% endif -%} += inner = +1 101 201 +2 102 202 +3 103 203 +4 104 204 +5 105 205 += full = +{% if join_algorithm not in ['grace_hash'] -%} +0 0 -4 +0 0 -3 +0 0 -2 +0 0 -1 +0 0 0 +1 1 1 +2 2 2 +3 3 3 +4 4 4 +5 5 5 +6 6 0 +7 7 0 +8 8 0 +9 9 0 +10 10 0 +{% endif -%} += left = +1 1 1 +2 2 2 +3 3 3 +4 4 4 +5 5 5 +6 6 0 +7 7 0 +8 8 0 +9 9 0 +10 10 0 += right = +{% if join_algorithm not in ['grace_hash'] -%} +0 0 -4 +0 0 -3 +0 0 -2 +0 0 -1 +0 0 0 +1 1 1 +2 2 2 +3 3 3 +4 4 4 +5 5 5 +{% endif -%} += inner = +1 1 1 +2 2 2 +3 3 3 +4 4 4 +5 5 5 += join on = += full = +{% if join_algorithm not in ['grace_hash'] -%} +0 0 -4 196 +0 0 -3 197 +0 0 -2 198 +0 0 -1 199 +0 0 0 200 +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +6 106 0 \N +7 107 0 \N +8 108 0 \N +9 109 0 \N +10 110 0 \N +{% endif -%} += left = +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +6 106 0 \N +7 107 0 \N +8 108 0 \N +9 109 0 \N +10 110 0 \N += right = +{% if join_algorithm not in ['grace_hash'] -%} +0 0 -4 196 +0 0 -3 197 +0 0 -2 198 +0 0 -1 199 +0 0 0 200 +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +{% endif -%} += inner = +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 += full = +{% if join_algorithm not in ['grace_hash'] -%} +0 0 -4 196 +0 0 -3 197 +0 0 -2 198 +0 0 -1 199 +0 0 0 200 +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +6 106 0 \N +7 107 0 \N +8 108 0 \N +9 109 0 \N +10 110 0 \N +{% endif -%} += left = +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +6 106 0 \N +7 107 0 \N +8 108 0 \N +9 109 0 \N +10 110 0 \N += right = +{% if join_algorithm not in ['grace_hash'] -%} +0 0 -4 196 +0 0 -3 197 +0 0 -2 198 +0 0 -1 199 +0 0 0 200 +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +{% endif -%} += inner = +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 += agg = +1 +1 +{% if join_algorithm not in ['grace_hash'] -%} +1 +1 +1 +1 +0 -10 0 +1 55 1055 +0 0 -10 0 990 +1 55 15 1055 1015 +{% endif -%} += types = +1 +1 +1 +1 +{% if join_algorithm not in ['grace_hash'] -%} +1 +1 +1 +1 +1 +1 +1 +{% endif -%} +{% if join_algorithm not in ['full_sorting_merge'] -%} +=== join use nulls === += full = +{% if join_algorithm not in ['grace_hash'] -%} +-4 \N 196 +-3 \N 197 +-2 \N 198 +-1 \N 199 +0 \N 200 +1 101 201 +2 102 202 +3 103 203 +4 104 204 +5 105 205 +6 106 \N +7 107 \N +8 108 \N +9 109 \N +10 110 \N +{% endif -%} += left = +1 101 201 +2 102 202 +3 103 203 +4 104 204 +5 105 205 +6 106 \N +7 107 \N +8 108 \N +9 109 \N +10 110 \N += right = +{% if join_algorithm not in ['grace_hash'] -%} +-4 \N 196 +-3 \N 197 +-2 \N 198 +-1 \N 199 +0 \N 200 +1 101 201 +2 102 202 +3 103 203 +4 104 204 +5 105 205 +{% endif -%} += inner = +1 101 201 +2 102 202 +3 103 203 +4 104 204 +5 105 205 += full = +{% if join_algorithm not in ['grace_hash'] -%} +1 1 1 +2 2 2 +3 3 3 +4 4 4 +5 5 5 +6 6 \N +7 7 \N +8 8 \N +9 9 \N +10 10 \N +\N \N -4 +\N \N -3 +\N \N -2 +\N \N -1 +\N \N 0 +{% endif -%} += left = +1 1 1 +2 2 2 +3 3 3 +4 4 4 +5 5 5 +6 6 \N +7 7 \N +8 8 \N +9 9 \N +10 10 \N += right = +{% if join_algorithm not in ['grace_hash'] -%} +1 1 1 +2 2 2 +3 3 3 +4 4 4 +5 5 5 +\N \N -4 +\N \N -3 +\N \N -2 +\N \N -1 +\N \N 0 +{% endif -%} += inner = +1 1 1 +2 2 2 +3 3 3 +4 4 4 +5 5 5 += join on = += full = +{% if join_algorithm not in ['grace_hash'] -%} +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +6 106 \N \N +7 107 \N \N +8 108 \N \N +9 109 \N \N +10 110 \N \N +\N \N -4 196 +\N \N -3 197 +\N \N -2 198 +\N \N -1 199 +\N \N 0 200 +{% endif -%} += left = +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +6 106 \N \N +7 107 \N \N +8 108 \N \N +9 109 \N \N +10 110 \N \N += right = +{% if join_algorithm not in ['grace_hash'] -%} +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +\N \N -4 196 +\N \N -3 197 +\N \N -2 198 +\N \N -1 199 +\N \N 0 200 +{% endif -%} += inner = +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 += full = +{% if join_algorithm not in ['grace_hash'] -%} +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +6 106 \N \N +7 107 \N \N +8 108 \N \N +9 109 \N \N +10 110 \N \N +\N \N -4 196 +\N \N -3 197 +\N \N -2 198 +\N \N -1 199 +\N \N 0 200 +{% endif -%} += left = +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +6 106 \N \N +7 107 \N \N +8 108 \N \N +9 109 \N \N +10 110 \N \N += right = +{% if join_algorithm not in ['grace_hash'] -%} +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 +\N \N -4 196 +\N \N -3 197 +\N \N -2 198 +\N \N -1 199 +\N \N 0 200 +{% endif -%} += inner = +1 101 1 201 +2 102 2 202 +3 103 3 203 +4 104 4 204 +5 105 5 205 += agg = +1 +1 +{% if join_algorithm not in ['grace_hash'] -%} +1 +1 +1 +1 +0 -10 \N +1 55 1055 +1 55 15 1055 1015 +\N \N -10 \N 990 +{% endif -%} += types = +1 +1 +1 +1 +{% if join_algorithm not in ['grace_hash'] -%} +1 +1 +1 +1 +1 +1 +1 +{% endif -%} +{% endif -%} +{% endfor -%} diff --git a/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 b/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 index 3846f527bba..f5321939f28 100644 --- a/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 +++ b/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 @@ -9,49 +9,55 @@ CREATE TABLE t2 (a Int16, b Nullable(Int64)) ENGINE = TinyLog; INSERT INTO t1 SELECT number as a, 100 + number as b FROM system.numbers LIMIT 1, 10; INSERT INTO t2 SELECT number - 5 as a, 200 + number - 5 as b FROM system.numbers LIMIT 1, 10; -{% for join_type in ['hash', 'partial_merge', 'full_sorting_merge', 'auto'] -%} +{% macro is_implemented(join_algorithm) -%} +{% if join_algorithm == 'grace_hash' %} -- { serverError NOT_IMPLEMENTED } {% endif %} +{% endmacro -%} -SELECT '=== {{ join_type }} ==='; -SET join_algorithm = '{{ join_type }}'; +{% for join_algorithm in ['hash', 'partial_merge', 'auto', 'full_sorting_merge', 'grace_hash'] -%} -{% if join_type == 'auto' -%} +SELECT '=== {{ join_algorithm }} ==='; +SET join_algorithm = '{{ join_algorithm }}'; + +{% if join_algorithm == 'auto' -%} SET max_bytes_in_join = 100; +{% else %} +SET max_bytes_in_join = '100M'; {% endif -%} SELECT '= full ='; -SELECT a, b, t2.b FROM t1 FULL JOIN t2 USING (a) ORDER BY (a); +SELECT a, b, t2.b FROM t1 FULL JOIN t2 USING (a) ORDER BY (a); {{ is_implemented(join_algorithm) }} SELECT '= left ='; SELECT a, b, t2.b FROM t1 LEFT JOIN t2 USING (a) ORDER BY (a); SELECT '= right ='; -SELECT a, b, t2.b FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (a); +SELECT a, b, t2.b FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (a); {{ is_implemented(join_algorithm) }} SELECT '= inner ='; SELECT a, b, t2.b FROM t1 INNER JOIN t2 USING (a) ORDER BY (a); SELECT '= full ='; -SELECT a, t1.a, t2.a FROM t1 FULL JOIN t2 USING (a) ORDER BY (t1.a, t2.a); +SELECT a, t1.a, t2.a FROM t1 FULL JOIN t2 USING (a) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= left ='; SELECT a, t1.a, t2.a FROM t1 LEFT JOIN t2 USING (a) ORDER BY (t1.a, t2.a); SELECT '= right ='; -SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a); +SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= inner ='; SELECT a, t1.a, t2.a FROM t1 INNER JOIN t2 USING (a) ORDER BY (t1.a, t2.a); SELECT '= join on ='; SELECT '= full ='; -SELECT a, b, t2.a, t2.b FROM t1 FULL JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); +SELECT a, b, t2.a, t2.b FROM t1 FULL JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= left ='; SELECT a, b, t2.a, t2.b FROM t1 LEFT JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); SELECT '= right ='; -SELECT a, b, t2.a, t2.b FROM t1 RIGHT JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); +SELECT a, b, t2.a, t2.b FROM t1 RIGHT JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= inner ='; SELECT a, b, t2.a, t2.b FROM t1 INNER JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); SELECT '= full ='; -SELECT * FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); +SELECT * FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= left ='; SELECT * FROM t1 LEFT JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); SELECT '= right ='; -SELECT * FROM t1 RIGHT JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); +SELECT * FROM t1 RIGHT JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= inner ='; SELECT * FROM t1 INNER JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); @@ -62,77 +68,77 @@ SELECT * FROM t1 RIGHT JOIN t2 ON (t1.a + t1.b + 100 = t2.a + t2.b) ORDER BY (t1 SELECT * FROM t1 INNER JOIN t2 ON (t1.a + t1.b + 100 = t2.a + t2.b) ORDER BY (t1.a, t2.a); -- { serverError 53 } SELECT '= agg ='; -SELECT sum(a) == 7 FROM t1 FULL JOIN t2 USING (a) WHERE b > 102 AND t2.b <= 204; +SELECT sum(a) == 7 FROM t1 FULL JOIN t2 USING (a) WHERE b > 102 AND t2.b <= 204; {{ is_implemented(join_algorithm) }} SELECT sum(a) == 7 FROM t1 INNER JOIN t2 USING (a) WHERE b > 102 AND t2.b <= 204; SELECT sum(b) = 103 FROM t1 LEFT JOIN t2 USING (a) WHERE b > 102 AND t2.b < 204; -SELECT sum(t2.b) = 203 FROM t1 RIGHT JOIN t2 USING (a) WHERE b > 102 AND t2.b < 204; +SELECT sum(t2.b) = 203 FROM t1 RIGHT JOIN t2 USING (a) WHERE b > 102 AND t2.b < 204; {{ is_implemented(join_algorithm) }} -SELECT sum(a) == 2 + 3 + 4 FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) WHERE t1.b < 105 AND t2.b > 201; -SELECT sum(a) == 55 FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) WHERE 1; +SELECT sum(a) == 2 + 3 + 4 FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) WHERE t1.b < 105 AND t2.b > 201; {{ is_implemented(join_algorithm) }} +SELECT sum(a) == 55 FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) WHERE 1; {{ is_implemented(join_algorithm) }} -SELECT a > 0, sum(a), sum(b) FROM t1 FULL JOIN t2 USING (a) GROUP BY (a > 0) ORDER BY a > 0; -SELECT a > 0, sum(a), sum(t2.a), sum(b), sum(t2.b) FROM t1 FULL JOIN t2 ON (t1.a == t2.a) GROUP BY (a > 0) ORDER BY a > 0; +SELECT a > 0, sum(a), sum(b) FROM t1 FULL JOIN t2 USING (a) GROUP BY (a > 0) ORDER BY a > 0; {{ is_implemented(join_algorithm) }} +SELECT a > 0, sum(a), sum(t2.a), sum(b), sum(t2.b) FROM t1 FULL JOIN t2 ON (t1.a == t2.a) GROUP BY (a > 0) ORDER BY a > 0; {{ is_implemented(join_algorithm) }} SELECT '= types ='; -SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 FULL JOIN t2 USING (a); +SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 FULL JOIN t2 USING (a); {{ is_implemented(join_algorithm) }} SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 LEFT JOIN t2 USING (a); -SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 RIGHT JOIN t2 USING (a); +SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 RIGHT JOIN t2 USING (a); {{ is_implemented(join_algorithm) }} SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 INNER JOIN t2 USING (a); -SELECT toTypeName(any(a)) == 'Int32' AND toTypeName(any(t2.a)) == 'Int32' FROM t1 FULL JOIN t2 USING (a); -SELECT min(toTypeName(a) == 'Int32' AND toTypeName(t2.a) == 'Int32') FROM t1 FULL JOIN t2 USING (a); +SELECT toTypeName(any(a)) == 'Int32' AND toTypeName(any(t2.a)) == 'Int32' FROM t1 FULL JOIN t2 USING (a); {{ is_implemented(join_algorithm) }} +SELECT min(toTypeName(a) == 'Int32' AND toTypeName(t2.a) == 'Int32') FROM t1 FULL JOIN t2 USING (a); {{ is_implemented(join_algorithm) }} -SELECT any(toTypeName(a)) == 'UInt16' AND any(toTypeName(t2.a)) == 'Int16' FROM t1 FULL JOIN t2 ON (t1.a == t2.a); +SELECT any(toTypeName(a)) == 'UInt16' AND any(toTypeName(t2.a)) == 'Int16' FROM t1 FULL JOIN t2 ON (t1.a == t2.a); {{ is_implemented(join_algorithm) }} SELECT any(toTypeName(a)) == 'UInt16' AND any(toTypeName(t2.a)) == 'Int16' FROM t1 LEFT JOIN t2 ON (t1.a == t2.a); -SELECT any(toTypeName(a)) == 'UInt16' AND any(toTypeName(t2.a)) == 'Int16' FROM t1 RIGHT JOIN t2 ON (t1.a == t2.a); +SELECT any(toTypeName(a)) == 'UInt16' AND any(toTypeName(t2.a)) == 'Int16' FROM t1 RIGHT JOIN t2 ON (t1.a == t2.a); {{ is_implemented(join_algorithm) }} SELECT any(toTypeName(a)) == 'UInt16' AND any(toTypeName(t2.a)) == 'Int16' FROM t1 INNER JOIN t2 ON (t1.a == t2.a); -SELECT toTypeName(any(a)) == 'UInt16' AND toTypeName(any(t2.a)) == 'Int16' FROM t1 FULL JOIN t2 ON (t1.a == t2.a); +SELECT toTypeName(any(a)) == 'UInt16' AND toTypeName(any(t2.a)) == 'Int16' FROM t1 FULL JOIN t2 ON (t1.a == t2.a); {{ is_implemented(join_algorithm) }} -{% if join_type == 'auto' -%} +{% if join_algorithm == 'auto' -%} SET max_bytes_in_join = 0; {% endif -%} -{% endfor -%} +{% if join_algorithm not in ['full_sorting_merge'] -%} SELECT '=== join use nulls ==='; SET join_use_nulls = 1; SELECT '= full ='; -SELECT a, b, t2.b FROM t1 FULL JOIN t2 USING (a) ORDER BY (a); +SELECT a, b, t2.b FROM t1 FULL JOIN t2 USING (a) ORDER BY (a); {{ is_implemented(join_algorithm) }} SELECT '= left ='; SELECT a, b, t2.b FROM t1 LEFT JOIN t2 USING (a) ORDER BY (a); SELECT '= right ='; -SELECT a, b, t2.b FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (a); +SELECT a, b, t2.b FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (a); {{ is_implemented(join_algorithm) }} SELECT '= inner ='; SELECT a, b, t2.b FROM t1 INNER JOIN t2 USING (a) ORDER BY (a); SELECT '= full ='; -SELECT a, t1.a, t2.a FROM t1 FULL JOIN t2 USING (a) ORDER BY (t1.a, t2.a); +SELECT a, t1.a, t2.a FROM t1 FULL JOIN t2 USING (a) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= left ='; SELECT a, t1.a, t2.a FROM t1 LEFT JOIN t2 USING (a) ORDER BY (t1.a, t2.a); SELECT '= right ='; -SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a); +SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= inner ='; SELECT a, t1.a, t2.a FROM t1 INNER JOIN t2 USING (a) ORDER BY (t1.a, t2.a); SELECT '= join on ='; SELECT '= full ='; -SELECT a, b, t2.a, t2.b FROM t1 FULL JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); +SELECT a, b, t2.a, t2.b FROM t1 FULL JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= left ='; SELECT a, b, t2.a, t2.b FROM t1 LEFT JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); SELECT '= right ='; -SELECT a, b, t2.a, t2.b FROM t1 RIGHT JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); +SELECT a, b, t2.a, t2.b FROM t1 RIGHT JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= inner ='; SELECT a, b, t2.a, t2.b FROM t1 INNER JOIN t2 ON (t1.a == t2.a) ORDER BY (t1.a, t2.a); SELECT '= full ='; -SELECT * FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); +SELECT * FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= left ='; SELECT * FROM t1 LEFT JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); SELECT '= right ='; -SELECT * FROM t1 RIGHT JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); +SELECT * FROM t1 RIGHT JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); {{ is_implemented(join_algorithm) }} SELECT '= inner ='; SELECT * FROM t1 INNER JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) ORDER BY (t1.a, t2.a); @@ -143,34 +149,37 @@ SELECT * FROM t1 RIGHT JOIN t2 ON (t1.a + t1.b + 100 = t2.a + t2.b) ORDER BY (t1 SELECT * FROM t1 INNER JOIN t2 ON (t1.a + t1.b + 100 = t2.a + t2.b) ORDER BY (t1.a, t2.a); -- { serverError 53 } SELECT '= agg ='; -SELECT sum(a) == 7 FROM t1 FULL JOIN t2 USING (a) WHERE b > 102 AND t2.b <= 204; +SELECT sum(a) == 7 FROM t1 FULL JOIN t2 USING (a) WHERE b > 102 AND t2.b <= 204; {{ is_implemented(join_algorithm) }} SELECT sum(a) == 7 FROM t1 INNER JOIN t2 USING (a) WHERE b > 102 AND t2.b <= 204; SELECT sum(b) = 103 FROM t1 LEFT JOIN t2 USING (a) WHERE b > 102 AND t2.b < 204; -SELECT sum(t2.b) = 203 FROM t1 RIGHT JOIN t2 USING (a) WHERE b > 102 AND t2.b < 204; +SELECT sum(t2.b) = 203 FROM t1 RIGHT JOIN t2 USING (a) WHERE b > 102 AND t2.b < 204; {{ is_implemented(join_algorithm) }} -SELECT sum(a) == 2 + 3 + 4 FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) WHERE t1.b < 105 AND t2.b > 201; -SELECT sum(a) == 55 FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) WHERE 1; +SELECT sum(a) == 2 + 3 + 4 FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) WHERE t1.b < 105 AND t2.b > 201; {{ is_implemented(join_algorithm) }} +SELECT sum(a) == 55 FROM t1 FULL JOIN t2 ON (t1.a + t1.b = t2.a + t2.b - 100) WHERE 1; {{ is_implemented(join_algorithm) }} -SELECT a > 0, sum(a), sum(b) FROM t1 FULL JOIN t2 USING (a) GROUP BY (a > 0) ORDER BY a > 0; -SELECT a > 0, sum(a), sum(t2.a), sum(b), sum(t2.b) FROM t1 FULL JOIN t2 ON (t1.a == t2.a) GROUP BY (a > 0) ORDER BY a > 0; +SELECT a > 0, sum(a), sum(b) FROM t1 FULL JOIN t2 USING (a) GROUP BY (a > 0) ORDER BY a > 0; {{ is_implemented(join_algorithm) }} +SELECT a > 0, sum(a), sum(t2.a), sum(b), sum(t2.b) FROM t1 FULL JOIN t2 ON (t1.a == t2.a) GROUP BY (a > 0) ORDER BY a > 0; {{ is_implemented(join_algorithm) }} SELECT '= types ='; -SELECT any(toTypeName(a)) == 'Nullable(Int32)' AND any(toTypeName(t2.a)) == 'Nullable(Int32)' FROM t1 FULL JOIN t2 USING (a); +SELECT any(toTypeName(a)) == 'Nullable(Int32)' AND any(toTypeName(t2.a)) == 'Nullable(Int32)' FROM t1 FULL JOIN t2 USING (a); {{ is_implemented(join_algorithm) }} SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Nullable(Int32)' FROM t1 LEFT JOIN t2 USING (a); -SELECT any(toTypeName(a)) == 'Nullable(Int32)' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 RIGHT JOIN t2 USING (a); +SELECT any(toTypeName(a)) == 'Nullable(Int32)' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 RIGHT JOIN t2 USING (a); {{ is_implemented(join_algorithm) }} SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 INNER JOIN t2 USING (a); -SELECT toTypeName(any(a)) == 'Nullable(Int32)' AND toTypeName(any(t2.a)) == 'Nullable(Int32)' FROM t1 FULL JOIN t2 USING (a); -SELECT min(toTypeName(a) == 'Nullable(Int32)' AND toTypeName(t2.a) == 'Nullable(Int32)') FROM t1 FULL JOIN t2 USING (a); +SELECT toTypeName(any(a)) == 'Nullable(Int32)' AND toTypeName(any(t2.a)) == 'Nullable(Int32)' FROM t1 FULL JOIN t2 USING (a); {{ is_implemented(join_algorithm) }} +SELECT min(toTypeName(a) == 'Nullable(Int32)' AND toTypeName(t2.a) == 'Nullable(Int32)') FROM t1 FULL JOIN t2 USING (a); {{ is_implemented(join_algorithm) }} -SELECT any(toTypeName(a)) == 'Nullable(UInt16)' AND any(toTypeName(t2.a)) == 'Nullable(Int16)' FROM t1 FULL JOIN t2 ON (t1.a == t2.a); +SELECT any(toTypeName(a)) == 'Nullable(UInt16)' AND any(toTypeName(t2.a)) == 'Nullable(Int16)' FROM t1 FULL JOIN t2 ON (t1.a == t2.a); {{ is_implemented(join_algorithm) }} SELECT any(toTypeName(a)) == 'UInt16' AND any(toTypeName(t2.a)) == 'Nullable(Int16)' FROM t1 LEFT JOIN t2 ON (t1.a == t2.a); -SELECT any(toTypeName(a)) == 'Nullable(UInt16)' AND any(toTypeName(t2.a)) == 'Int16' FROM t1 RIGHT JOIN t2 ON (t1.a == t2.a); +SELECT any(toTypeName(a)) == 'Nullable(UInt16)' AND any(toTypeName(t2.a)) == 'Int16' FROM t1 RIGHT JOIN t2 ON (t1.a == t2.a); {{ is_implemented(join_algorithm) }} SELECT any(toTypeName(a)) == 'UInt16' AND any(toTypeName(t2.a)) == 'Int16' FROM t1 INNER JOIN t2 ON (t1.a == t2.a); -SELECT toTypeName(any(a)) == 'Nullable(UInt16)' AND toTypeName(any(t2.a)) == 'Nullable(Int16)' FROM t1 FULL JOIN t2 ON (t1.a == t2.a); +SELECT toTypeName(any(a)) == 'Nullable(UInt16)' AND toTypeName(any(t2.a)) == 'Nullable(Int16)' FROM t1 FULL JOIN t2 ON (t1.a == t2.a); {{ is_implemented(join_algorithm) }} SET join_use_nulls = 0; +{% endif -%} + +{% endfor -%} DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; diff --git a/tests/queries/0_stateless/01801_s3_cluster_count.reference b/tests/queries/0_stateless/01801_s3_cluster_count.reference new file mode 100644 index 00000000000..c094c553f81 --- /dev/null +++ b/tests/queries/0_stateless/01801_s3_cluster_count.reference @@ -0,0 +1,2 @@ +12 +12 diff --git a/tests/queries/0_stateless/01801_s3_cluster_count.sql b/tests/queries/0_stateless/01801_s3_cluster_count.sql new file mode 100644 index 00000000000..8a4fb804967 --- /dev/null +++ b/tests/queries/0_stateless/01801_s3_cluster_count.sql @@ -0,0 +1,5 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: Depends on AWS + +select COUNT() from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv'); +select COUNT(*) from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv'); diff --git a/tests/queries/0_stateless/01825_type_json_1.reference b/tests/queries/0_stateless/01825_type_json_1.reference index 857c624fb9b..3f0eaf3854a 100644 --- a/tests/queries/0_stateless/01825_type_json_1.reference +++ b/tests/queries/0_stateless/01825_type_json_1.reference @@ -6,22 +6,26 @@ all_2_2_0 data Tuple(k5 String) all_1_2_1 data Tuple(k1 String, k2 Tuple(k3 String, k4 String), k5 String) ============ 1 ['aaa','ddd'] [['bbb','ccc'],['eee','fff']] +all_1_2_2 data Tuple(_dummy UInt8) all_3_3_0 data Tuple(k1 Nested(k2 String, k3 Nested(k4 String))) ============ 1 a 42 2 b 4200 4242 +all_1_2_3 data Tuple(_dummy UInt8) all_4_4_0 data Tuple(name String, value Int16) 1 a 42 2 b 4200 3 a 42.123 +all_1_2_3 data Tuple(_dummy UInt8) all_4_4_0 data Tuple(name String, value Int16) all_5_5_0 data Tuple(name String, value Float64) 1 a 42 2 b 4200 3 a 42.123 4 a some +all_1_2_3 data Tuple(_dummy UInt8) all_4_4_0 data Tuple(name String, value Int16) all_5_5_0 data Tuple(name String, value Float64) all_6_6_0 data Tuple(name String, value String) -all_4_6_1 data Tuple(name String, value String) +all_1_6_4 data Tuple(name String, value String) diff --git a/tests/queries/0_stateless/01825_type_json_17.sql b/tests/queries/0_stateless/01825_type_json_17.sql index e3c0c83322b..ee5cf590407 100644 --- a/tests/queries/0_stateless/01825_type_json_17.sql +++ b/tests/queries/0_stateless/01825_type_json_17.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-parallel DROP TABLE IF EXISTS t_json_17; SET allow_experimental_object_type = 1; diff --git a/tests/queries/0_stateless/01825_type_json_18.reference b/tests/queries/0_stateless/01825_type_json_18.reference new file mode 100644 index 00000000000..d93f9bda63c --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_18.reference @@ -0,0 +1,2 @@ +1 (1) Tuple(k1 Int8) +1 ([1,2]) Tuple(k1 Array(Int8)) diff --git a/tests/queries/0_stateless/01825_type_json_18.sql b/tests/queries/0_stateless/01825_type_json_18.sql new file mode 100644 index 00000000000..b493982a12c --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_18.sql @@ -0,0 +1,16 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + +DROP TABLE IF EXISTS t_json_2; + +CREATE TABLE t_json_2(id UInt64, data Object('JSON')) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_json_2 FORMAT JSONEachRow {"id": 1, "data" : {"k1": 1}}; +SELECT id, data, toTypeName(data) FROM t_json_2 ORDER BY id; + +TRUNCATE TABLE t_json_2; + +INSERT INTO t_json_2 FORMAT JSONEachRow {"id": 1, "data" : {"k1": [1, 2]}}; +SELECT id, data, toTypeName(data) FROM t_json_2 ORDER BY id; diff --git a/tests/queries/0_stateless/02028_system_data_skipping_indices_size.sql b/tests/queries/0_stateless/02028_system_data_skipping_indices_size.sql index e77f88aa36f..1efb9cff6a4 100644 --- a/tests/queries/0_stateless/02028_system_data_skipping_indices_size.sql +++ b/tests/queries/0_stateless/02028_system_data_skipping_indices_size.sql @@ -7,7 +7,7 @@ CREATE TABLE test_table INDEX value_index value TYPE minmax GRANULARITY 1 ) Engine=MergeTree() -ORDER BY key; +ORDER BY key SETTINGS compress_marks=false; INSERT INTO test_table VALUES (0, 'Value'); SELECT * FROM system.data_skipping_indices WHERE database = currentDatabase(); diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index ce881422f63..5033e888896 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -14,9 +14,7 @@ CREATE TABLE system.asynchronous_inserts `first_update` DateTime64(6), `total_bytes` UInt64, `entries.query_id` Array(String), - `entries.bytes` Array(UInt64), - `entries.finished` Array(UInt8), - `entries.exception` Array(String) + `entries.bytes` Array(UInt64) ) ENGINE = SystemAsynchronousInserts COMMENT 'SYSTEM TABLE is built on the fly.' @@ -286,7 +284,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'SHOW NAMED COLLECTIONS' = 88, 'ACCESS MANAGEMENT' = 89, 'SYSTEM SHUTDOWN' = 90, 'SYSTEM DROP DNS CACHE' = 91, 'SYSTEM DROP MARK CACHE' = 92, 'SYSTEM DROP UNCOMPRESSED CACHE' = 93, 'SYSTEM DROP MMAP CACHE' = 94, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 95, 'SYSTEM DROP FILESYSTEM CACHE' = 96, 'SYSTEM DROP SCHEMA CACHE' = 97, 'SYSTEM DROP CACHE' = 98, 'SYSTEM RELOAD CONFIG' = 99, 'SYSTEM RELOAD USERS' = 100, 'SYSTEM RELOAD SYMBOLS' = 101, 'SYSTEM RELOAD DICTIONARY' = 102, 'SYSTEM RELOAD MODEL' = 103, 'SYSTEM RELOAD FUNCTION' = 104, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 105, 'SYSTEM RELOAD' = 106, 'SYSTEM RESTART DISK' = 107, 'SYSTEM MERGES' = 108, 'SYSTEM TTL MERGES' = 109, 'SYSTEM FETCHES' = 110, 'SYSTEM MOVES' = 111, 'SYSTEM DISTRIBUTED SENDS' = 112, 'SYSTEM REPLICATED SENDS' = 113, 'SYSTEM SENDS' = 114, 'SYSTEM REPLICATION QUEUES' = 115, 'SYSTEM DROP REPLICA' = 116, 'SYSTEM SYNC REPLICA' = 117, 'SYSTEM RESTART REPLICA' = 118, 'SYSTEM RESTORE REPLICA' = 119, 'SYSTEM SYNC DATABASE REPLICA' = 120, 'SYSTEM SYNC TRANSACTION LOG' = 121, 'SYSTEM FLUSH DISTRIBUTED' = 122, 'SYSTEM FLUSH LOGS' = 123, 'SYSTEM FLUSH' = 124, 'SYSTEM THREAD FUZZER' = 125, 'SYSTEM UNFREEZE' = 126, 'SYSTEM' = 127, 'dictGet' = 128, 'addressToLine' = 129, 'addressToLineWithInlines' = 130, 'addressToSymbol' = 131, 'demangle' = 132, 'INTROSPECTION' = 133, 'FILE' = 134, 'URL' = 135, 'REMOTE' = 136, 'MONGO' = 137, 'MEILISEARCH' = 138, 'MYSQL' = 139, 'POSTGRES' = 140, 'SQLITE' = 141, 'ODBC' = 142, 'JDBC' = 143, 'HDFS' = 144, 'S3' = 145, 'HIVE' = 146, 'SOURCES' = 147, 'CLUSTER' = 148, 'ALL' = 149, 'NONE' = 150), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM SYNC DATABASE REPLICA' = 123, 'SYSTEM SYNC TRANSACTION LOG' = 124, 'SYSTEM FLUSH DISTRIBUTED' = 125, 'SYSTEM FLUSH LOGS' = 126, 'SYSTEM FLUSH' = 127, 'SYSTEM THREAD FUZZER' = 128, 'SYSTEM UNFREEZE' = 129, 'SYSTEM' = 130, 'dictGet' = 131, 'addressToLine' = 132, 'addressToLineWithInlines' = 133, 'addressToSymbol' = 134, 'demangle' = 135, 'INTROSPECTION' = 136, 'FILE' = 137, 'URL' = 138, 'REMOTE' = 139, 'MONGO' = 140, 'MEILISEARCH' = 141, 'MYSQL' = 142, 'POSTGRES' = 143, 'SQLITE' = 144, 'ODBC' = 145, 'JDBC' = 146, 'HDFS' = 147, 'S3' = 148, 'HIVE' = 149, 'SOURCES' = 150, 'CLUSTER' = 151, 'ALL' = 152, 'NONE' = 153), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -562,10 +560,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'SHOW NAMED COLLECTIONS' = 88, 'ACCESS MANAGEMENT' = 89, 'SYSTEM SHUTDOWN' = 90, 'SYSTEM DROP DNS CACHE' = 91, 'SYSTEM DROP MARK CACHE' = 92, 'SYSTEM DROP UNCOMPRESSED CACHE' = 93, 'SYSTEM DROP MMAP CACHE' = 94, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 95, 'SYSTEM DROP FILESYSTEM CACHE' = 96, 'SYSTEM DROP SCHEMA CACHE' = 97, 'SYSTEM DROP CACHE' = 98, 'SYSTEM RELOAD CONFIG' = 99, 'SYSTEM RELOAD USERS' = 100, 'SYSTEM RELOAD SYMBOLS' = 101, 'SYSTEM RELOAD DICTIONARY' = 102, 'SYSTEM RELOAD MODEL' = 103, 'SYSTEM RELOAD FUNCTION' = 104, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 105, 'SYSTEM RELOAD' = 106, 'SYSTEM RESTART DISK' = 107, 'SYSTEM MERGES' = 108, 'SYSTEM TTL MERGES' = 109, 'SYSTEM FETCHES' = 110, 'SYSTEM MOVES' = 111, 'SYSTEM DISTRIBUTED SENDS' = 112, 'SYSTEM REPLICATED SENDS' = 113, 'SYSTEM SENDS' = 114, 'SYSTEM REPLICATION QUEUES' = 115, 'SYSTEM DROP REPLICA' = 116, 'SYSTEM SYNC REPLICA' = 117, 'SYSTEM RESTART REPLICA' = 118, 'SYSTEM RESTORE REPLICA' = 119, 'SYSTEM SYNC DATABASE REPLICA' = 120, 'SYSTEM SYNC TRANSACTION LOG' = 121, 'SYSTEM FLUSH DISTRIBUTED' = 122, 'SYSTEM FLUSH LOGS' = 123, 'SYSTEM FLUSH' = 124, 'SYSTEM THREAD FUZZER' = 125, 'SYSTEM UNFREEZE' = 126, 'SYSTEM' = 127, 'dictGet' = 128, 'addressToLine' = 129, 'addressToLineWithInlines' = 130, 'addressToSymbol' = 131, 'demangle' = 132, 'INTROSPECTION' = 133, 'FILE' = 134, 'URL' = 135, 'REMOTE' = 136, 'MONGO' = 137, 'MEILISEARCH' = 138, 'MYSQL' = 139, 'POSTGRES' = 140, 'SQLITE' = 141, 'ODBC' = 142, 'JDBC' = 143, 'HDFS' = 144, 'S3' = 145, 'HIVE' = 146, 'SOURCES' = 147, 'CLUSTER' = 148, 'ALL' = 149, 'NONE' = 150), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM SYNC DATABASE REPLICA' = 123, 'SYSTEM SYNC TRANSACTION LOG' = 124, 'SYSTEM FLUSH DISTRIBUTED' = 125, 'SYSTEM FLUSH LOGS' = 126, 'SYSTEM FLUSH' = 127, 'SYSTEM THREAD FUZZER' = 128, 'SYSTEM UNFREEZE' = 129, 'SYSTEM' = 130, 'dictGet' = 131, 'addressToLine' = 132, 'addressToLineWithInlines' = 133, 'addressToSymbol' = 134, 'demangle' = 135, 'INTROSPECTION' = 136, 'FILE' = 137, 'URL' = 138, 'REMOTE' = 139, 'MONGO' = 140, 'MEILISEARCH' = 141, 'MYSQL' = 142, 'POSTGRES' = 143, 'SQLITE' = 144, 'ODBC' = 145, 'JDBC' = 146, 'HDFS' = 147, 'S3' = 148, 'HIVE' = 149, 'SOURCES' = 150, 'CLUSTER' = 151, 'ALL' = 152, 'NONE' = 153), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'SHOW NAMED COLLECTIONS' = 88, 'ACCESS MANAGEMENT' = 89, 'SYSTEM SHUTDOWN' = 90, 'SYSTEM DROP DNS CACHE' = 91, 'SYSTEM DROP MARK CACHE' = 92, 'SYSTEM DROP UNCOMPRESSED CACHE' = 93, 'SYSTEM DROP MMAP CACHE' = 94, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 95, 'SYSTEM DROP FILESYSTEM CACHE' = 96, 'SYSTEM DROP SCHEMA CACHE' = 97, 'SYSTEM DROP CACHE' = 98, 'SYSTEM RELOAD CONFIG' = 99, 'SYSTEM RELOAD USERS' = 100, 'SYSTEM RELOAD SYMBOLS' = 101, 'SYSTEM RELOAD DICTIONARY' = 102, 'SYSTEM RELOAD MODEL' = 103, 'SYSTEM RELOAD FUNCTION' = 104, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 105, 'SYSTEM RELOAD' = 106, 'SYSTEM RESTART DISK' = 107, 'SYSTEM MERGES' = 108, 'SYSTEM TTL MERGES' = 109, 'SYSTEM FETCHES' = 110, 'SYSTEM MOVES' = 111, 'SYSTEM DISTRIBUTED SENDS' = 112, 'SYSTEM REPLICATED SENDS' = 113, 'SYSTEM SENDS' = 114, 'SYSTEM REPLICATION QUEUES' = 115, 'SYSTEM DROP REPLICA' = 116, 'SYSTEM SYNC REPLICA' = 117, 'SYSTEM RESTART REPLICA' = 118, 'SYSTEM RESTORE REPLICA' = 119, 'SYSTEM SYNC DATABASE REPLICA' = 120, 'SYSTEM SYNC TRANSACTION LOG' = 121, 'SYSTEM FLUSH DISTRIBUTED' = 122, 'SYSTEM FLUSH LOGS' = 123, 'SYSTEM FLUSH' = 124, 'SYSTEM THREAD FUZZER' = 125, 'SYSTEM UNFREEZE' = 126, 'SYSTEM' = 127, 'dictGet' = 128, 'addressToLine' = 129, 'addressToLineWithInlines' = 130, 'addressToSymbol' = 131, 'demangle' = 132, 'INTROSPECTION' = 133, 'FILE' = 134, 'URL' = 135, 'REMOTE' = 136, 'MONGO' = 137, 'MEILISEARCH' = 138, 'MYSQL' = 139, 'POSTGRES' = 140, 'SQLITE' = 141, 'ODBC' = 142, 'JDBC' = 143, 'HDFS' = 144, 'S3' = 145, 'HIVE' = 146, 'SOURCES' = 147, 'CLUSTER' = 148, 'ALL' = 149, 'NONE' = 150)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM SYNC DATABASE REPLICA' = 123, 'SYSTEM SYNC TRANSACTION LOG' = 124, 'SYSTEM FLUSH DISTRIBUTED' = 125, 'SYSTEM FLUSH LOGS' = 126, 'SYSTEM FLUSH' = 127, 'SYSTEM THREAD FUZZER' = 128, 'SYSTEM UNFREEZE' = 129, 'SYSTEM' = 130, 'dictGet' = 131, 'addressToLine' = 132, 'addressToLineWithInlines' = 133, 'addressToSymbol' = 134, 'demangle' = 135, 'INTROSPECTION' = 136, 'FILE' = 137, 'URL' = 138, 'REMOTE' = 139, 'MONGO' = 140, 'MEILISEARCH' = 141, 'MYSQL' = 142, 'POSTGRES' = 143, 'SQLITE' = 144, 'ODBC' = 145, 'JDBC' = 146, 'HDFS' = 147, 'S3' = 148, 'HIVE' = 149, 'SOURCES' = 150, 'CLUSTER' = 151, 'ALL' = 152, 'NONE' = 153)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' diff --git a/tests/queries/0_stateless/02124_buffer_insert_select_race.reference b/tests/queries/0_stateless/02124_buffer_insert_select_race.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02124_buffer_insert_select_race.sh b/tests/queries/0_stateless/02124_buffer_insert_select_race.sh new file mode 100755 index 00000000000..22965a274c0 --- /dev/null +++ b/tests/queries/0_stateless/02124_buffer_insert_select_race.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# Regression test for 'Logical error: No column to rollback' in case of +# exception while commiting batch into the Buffer, see [1]. +# +# [1]: https://github.com/ClickHouse/ClickHouse/issues/42740 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_buffer_string" +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_buffer_string(key String) ENGINE = Buffer('', '', 1, 1, 1, 1000000000000, 1000000000000, 1000000000000, 1000000000000)" + +# --continue_on_errors -- to ignore possible MEMORY_LIMIT_EXCEEDED errors +# --concurrency -- we need have SELECT and INSERT in parallel to have refcount +# of the column in the Buffer block > 1, that way we will do +# full clone and moving a column may throw. +# +# It reproduces the problem 100% with MemoryTrackerFaultInjectorInThread in the appendBlock() +$CLICKHOUSE_BENCHMARK --randomize --timelimit 10 --continue_on_errors --concurrency 10 >& /dev/null < /dev/null > /dev/null - done -} - -TIMEOUT=10 - -export -f insert1 -export -f select1 - -timeout $TIMEOUT bash -c insert1 & -timeout $TIMEOUT bash -c select1 & - -wait +# --continue_on_errors -- to ignore possible MEMORY_LIMIT_EXCEEDED errors +$CLICKHOUSE_BENCHMARK --randomize --timelimit 10 --continue_on_errors --concurrency 10 >& /dev/null < x LIKE '%async_inserts_2156', tables), \ - query_kind, Settings['async_insert'], Settings['wait_for_async_insert'] FROM system.query_log \ + query_kind, Settings['async_insert'] FROM system.query_log \ WHERE event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' \ AND query ILIKE 'INSERT INTO async_inserts_2156 VALUES%' AND type = 'QueryFinish' \ ORDER BY query_start_time_microseconds" diff --git a/tests/queries/0_stateless/02240_filesystem_cache_bypass_cache_threshold.reference b/tests/queries/0_stateless/02240_filesystem_cache_bypass_cache_threshold.reference index de9ac10f641..997105c9da3 100644 --- a/tests/queries/0_stateless/02240_filesystem_cache_bypass_cache_threshold.reference +++ b/tests/queries/0_stateless/02240_filesystem_cache_bypass_cache_threshold.reference @@ -3,7 +3,7 @@ SYSTEM DROP FILESYSTEM CACHE; SET enable_filesystem_cache_on_write_operations=0; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_6', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_6', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; INSERT INTO test SELECT number, toString(number) FROM numbers(100); SELECT * FROM test FORMAT Null; SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_end, size; diff --git a/tests/queries/0_stateless/02240_filesystem_cache_bypass_cache_threshold.sql b/tests/queries/0_stateless/02240_filesystem_cache_bypass_cache_threshold.sql index d3b3d3d7f4c..f6671b82291 100644 --- a/tests/queries/0_stateless/02240_filesystem_cache_bypass_cache_threshold.sql +++ b/tests/queries/0_stateless/02240_filesystem_cache_bypass_cache_threshold.sql @@ -6,7 +6,7 @@ SYSTEM DROP FILESYSTEM CACHE; SET enable_filesystem_cache_on_write_operations=0; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_6', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_6', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; INSERT INTO test SELECT number, toString(number) FROM numbers(100); SELECT * FROM test FORMAT Null; diff --git a/tests/queries/0_stateless/02240_filesystem_query_cache.reference b/tests/queries/0_stateless/02240_filesystem_query_cache.reference index 329ca122af1..48d91c6f142 100644 --- a/tests/queries/0_stateless/02240_filesystem_query_cache.reference +++ b/tests/queries/0_stateless/02240_filesystem_query_cache.reference @@ -5,7 +5,7 @@ SET enable_filesystem_cache_on_write_operations=0; SET skip_download_if_exceeds_query_cache=1; SET max_query_cache_size=128; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_4', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_4', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; INSERT INTO test SELECT number, toString(number) FROM numbers(100); SELECT * FROM test FORMAT Null; SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_end, size; diff --git a/tests/queries/0_stateless/02240_filesystem_query_cache.sql b/tests/queries/0_stateless/02240_filesystem_query_cache.sql index 2a4f4ae219c..7dd975b27ee 100644 --- a/tests/queries/0_stateless/02240_filesystem_query_cache.sql +++ b/tests/queries/0_stateless/02240_filesystem_query_cache.sql @@ -8,7 +8,7 @@ SET skip_download_if_exceeds_query_cache=1; SET max_query_cache_size=128; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_4', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_4', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; INSERT INTO test SELECT number, toString(number) FROM numbers(100); SELECT * FROM test FORMAT Null; diff --git a/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference b/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference index c67eecf8cf2..6b96da0be59 100644 --- a/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference +++ b/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference @@ -4,7 +4,7 @@ Using storage policy: s3_cache SYSTEM DROP FILESYSTEM CACHE; SET enable_filesystem_cache_on_write_operations=0; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; INSERT INTO test SELECT number, toString(number) FROM numbers(100); SELECT * FROM test FORMAT Null; @@ -19,7 +19,7 @@ SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesy SYSTEM DROP FILESYSTEM CACHE; SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_3', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_3', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; INSERT INTO test SELECT number, toString(number) FROM numbers(100); SELECT * FROM test FORMAT Null; SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_end, size; @@ -39,7 +39,7 @@ Using storage policy: local_cache SYSTEM DROP FILESYSTEM CACHE; SET enable_filesystem_cache_on_write_operations=0; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; INSERT INTO test SELECT number, toString(number) FROM numbers(100); SELECT * FROM test FORMAT Null; @@ -54,7 +54,7 @@ SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesy SYSTEM DROP FILESYSTEM CACHE; SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache_3', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache_3', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; INSERT INTO test SELECT number, toString(number) FROM numbers(100); SELECT * FROM test FORMAT Null; SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_end, size; diff --git a/tests/queries/0_stateless/02241_array_first_last_or_null.reference b/tests/queries/0_stateless/02241_array_first_last_or_null.reference index 2906b04ecd0..fc4a5ff8af5 100644 --- a/tests/queries/0_stateless/02241_array_first_last_or_null.reference +++ b/tests/queries/0_stateless/02241_array_first_last_or_null.reference @@ -7,6 +7,9 @@ ArrayFirst non constant predicate \N 2 2 +ArrayFirst with Null +2 +\N ArrayLast constant predicate \N \N @@ -16,3 +19,6 @@ ArrayLast non constant predicate \N 3 3 +ArrayLast with Null +2 +\N diff --git a/tests/queries/0_stateless/02241_array_first_last_or_null.sql b/tests/queries/0_stateless/02241_array_first_last_or_null.sql index 3230e4d483a..aa8f0cdbf92 100644 --- a/tests/queries/0_stateless/02241_array_first_last_or_null.sql +++ b/tests/queries/0_stateless/02241_array_first_last_or_null.sql @@ -9,6 +9,10 @@ SELECT arrayFirstOrNull(x -> x >= 2, emptyArrayUInt8()); SELECT arrayFirstOrNull(x -> x >= 2, [1, 2, 3]); SELECT arrayFirstOrNull(x -> x >= 2, materialize([1, 2, 3])); +SELECT 'ArrayFirst with Null'; +SELECT arrayFirstOrNull((x,f) -> f, [1,2,3,NULL], [0,1,0,0]); +SELECT arrayFirstOrNull((x,f) -> f, [1,2,3,NULL], [0,0,0,1]); + SELECT 'ArrayLast constant predicate'; SELECT arrayLastOrNull(x -> 1, emptyArrayUInt8()); SELECT arrayLastOrNull(x -> 0, emptyArrayUInt8()); @@ -19,3 +23,7 @@ SELECT 'ArrayLast non constant predicate'; SELECT arrayLastOrNull(x -> x >= 2, emptyArrayUInt8()); SELECT arrayLastOrNull(x -> x >= 2, [1, 2, 3]); SELECT arrayLastOrNull(x -> x >= 2, materialize([1, 2, 3])); + +SELECT 'ArrayLast with Null'; +SELECT arrayLastOrNull((x,f) -> f, [1,2,3,NULL], [0,1,0,0]); +SELECT arrayLastOrNull((x,f) -> f, [1,2,3,NULL], [0,1,0,1]); \ No newline at end of file diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference index 5a1295db495..9405b9eb614 100644 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference @@ -3,7 +3,7 @@ Using storage policy: s3_cache SET enable_filesystem_cache_on_write_operations=1; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; SYSTEM DROP FILESYSTEM CACHE; SELECT file_segment_range_begin, file_segment_range_end, size, state @@ -129,7 +129,7 @@ Using storage policy: local_cache SET enable_filesystem_cache_on_write_operations=1; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; SYSTEM DROP FILESYSTEM CACHE; SELECT file_segment_range_begin, file_segment_range_end, size, state diff --git a/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.reference b/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.reference index 4a10ff02586..91587dc8e79 100644 --- a/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.reference +++ b/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.reference @@ -6,7 +6,7 @@ SET enable_filesystem_cache_log=1; SET enable_filesystem_cache_on_write_operations=0; DROP TABLE IF EXISTS test; DROP TABLE IF EXISTS system.filesystem_cache_log; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; INSERT INTO test SELECT number, toString(number) FROM numbers(100000); SELECT 2240, 's3_cache', * FROM test FORMAT Null; @@ -27,7 +27,7 @@ SET enable_filesystem_cache_log=1; SET enable_filesystem_cache_on_write_operations=0; DROP TABLE IF EXISTS test; DROP TABLE IF EXISTS system.filesystem_cache_log; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='local_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; INSERT INTO test SELECT number, toString(number) FROM numbers(100000); SELECT 2240, 'local_cache', * FROM test FORMAT Null; diff --git a/tests/queries/0_stateless/02245_s3_support_read_nested_column.reference b/tests/queries/0_stateless/02245_s3_support_read_nested_column.reference new file mode 100644 index 00000000000..e9754463ba1 --- /dev/null +++ b/tests/queries/0_stateless/02245_s3_support_read_nested_column.reference @@ -0,0 +1,31 @@ +-- { echo } +drop table if exists test_02245_s3_nested_parquet1; +drop table if exists test_02245_s3_nested_parquet2; +set input_format_parquet_import_nested = 1; +create table test_02245_s3_nested_parquet1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet1_{_partition_id}', format='Parquet') partition by a; +insert into test_02245_s3_nested_parquet1 values (1, (2, 'a')); +select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_parquet1_*', format='Parquet'); -- { serverError 47 } +create table test_02245_s3_nested_parquet2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet2_{_partition_id}', format='Parquet') partition by a; +insert into test_02245_s3_nested_parquet2 values (1, (2, (3, 'a'))); +select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_parquet2_*', format='Parquet', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); +1 2 3 a +drop table if exists test_02245_s3_nested_arrow1; +drop table if exists test_02245_s3_nested_arrow2; +set input_format_arrow_import_nested=1; +create table test_02245_s3_nested_arrow1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow1_{_partition_id}', format='Arrow') partition by a; +insert into test_02245_s3_nested_arrow1 values (1, (2, 'a')); +select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_arrow1_*', format='Arrow'); -- { serverError 47 } +create table test_02245_s3_nested_arrow2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow2_{_partition_id}', format='Arrow') partition by a; +insert into test_02245_s3_nested_arrow2 values (1, (2, (3, 'a'))); +select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_arrow2_*', format='Arrow', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); +1 2 3 a +drop table if exists test_02245_s3_nested_orc1; +drop table if exists test_02245_s3_nested_orc2; +set input_format_orc_import_nested=1; +create table test_02245_s3_nested_orc1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_orc1_{_partition_id}', format='ORC') partition by a; +insert into test_02245_s3_nested_orc1 values (1, (2, 'a')); +select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_orc1_*', format='ORC'); -- { serverError 47 } +create table test_02245_s3_nested_orc2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_orc2_{_partition_id}', format='ORC') partition by a; +insert into test_02245_s3_nested_orc2 values (1, (2, (3, 'a'))); +select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_orc2_*', format='ORC', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); +1 2 3 a diff --git a/tests/queries/0_stateless/02245_s3_support_read_nested_column.sql b/tests/queries/0_stateless/02245_s3_support_read_nested_column.sql new file mode 100644 index 00000000000..14fc7cee7dc --- /dev/null +++ b/tests/queries/0_stateless/02245_s3_support_read_nested_column.sql @@ -0,0 +1,44 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: Depends on AWS + +-- { echo } +drop table if exists test_02245_s3_nested_parquet1; +drop table if exists test_02245_s3_nested_parquet2; +set input_format_parquet_import_nested = 1; +create table test_02245_s3_nested_parquet1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet1_{_partition_id}', format='Parquet') partition by a; +insert into test_02245_s3_nested_parquet1 values (1, (2, 'a')); + +select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_parquet1_*', format='Parquet'); -- { serverError 47 } + +create table test_02245_s3_nested_parquet2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_parquet2_{_partition_id}', format='Parquet') partition by a; +insert into test_02245_s3_nested_parquet2 values (1, (2, (3, 'a'))); + +select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_parquet2_*', format='Parquet', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); + + +drop table if exists test_02245_s3_nested_arrow1; +drop table if exists test_02245_s3_nested_arrow2; +set input_format_arrow_import_nested=1; +create table test_02245_s3_nested_arrow1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow1_{_partition_id}', format='Arrow') partition by a; +insert into test_02245_s3_nested_arrow1 values (1, (2, 'a')); + +select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_arrow1_*', format='Arrow'); -- { serverError 47 } + +create table test_02245_s3_nested_arrow2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_arrow2_{_partition_id}', format='Arrow') partition by a; +insert into test_02245_s3_nested_arrow2 values (1, (2, (3, 'a'))); + +select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_arrow2_*', format='Arrow', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); + + +drop table if exists test_02245_s3_nested_orc1; +drop table if exists test_02245_s3_nested_orc2; +set input_format_orc_import_nested=1; +create table test_02245_s3_nested_orc1(a Int64, b Tuple(a Int64, b String)) engine=S3(s3_conn, filename='test_02245_s3_nested_orc1_{_partition_id}', format='ORC') partition by a; +insert into test_02245_s3_nested_orc1 values (1, (2, 'a')); + +select a, b.a, b.b from s3(s3_conn, filename='test_02245_s3_nested_orc1_*', format='ORC'); -- { serverError 47 } + +create table test_02245_s3_nested_orc2(a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))) engine=S3(s3_conn, filename='test_02245_s3_nested_orc2_{_partition_id}', format='ORC') partition by a; +insert into test_02245_s3_nested_orc2 values (1, (2, (3, 'a'))); + +select a, b.a, b.b.c, b.b.d from s3(s3_conn, filename='test_02245_s3_nested_orc2_*', format='ORC', structure='a Int64, b Tuple(a Int64, b Tuple(c Int64, d String))'); diff --git a/tests/queries/0_stateless/02273_full_sort_join.reference.j2 b/tests/queries/0_stateless/02273_full_sort_join.reference.j2 index 1059108a03b..98bfd9d9b2b 100644 --- a/tests/queries/0_stateless/02273_full_sort_join.reference.j2 +++ b/tests/queries/0_stateless/02273_full_sort_join.reference.j2 @@ -1,4 +1,6 @@ {% set table_size = 15 -%} +{% for join_algorithm in ['default', 'full_sorting_merge', 'grace_hash'] -%} +-- {{ join_algorithm }} -- {% for block_size in range(1, table_size + 1) -%} ALL INNER USING | bs = {{ block_size }} 4 0 0 @@ -48,6 +50,7 @@ ALL LEFT | bs = {{ block_size }} 14 14 val9 0 14 14 val9 0 ALL RIGHT | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 4 4 0 val10 5 5 0 val6 6 6 0 val8 @@ -61,6 +64,7 @@ ALL RIGHT | bs = {{ block_size }} 13 13 0 val9 14 14 0 val3 14 14 0 val7 +{% endif -%} ALL INNER | bs = {{ block_size }} | copmosite key 2 2 2 2 2 2 0 0 2 2 2 2 2 2 0 0 @@ -81,6 +85,7 @@ ALL LEFT | bs = {{ block_size }} | copmosite key 2 2 2 2 2 2 val12 0 2 2 2 2 2 2 val9 0 ALL RIGHT | bs = {{ block_size }} | copmosite key +{% if join_algorithm != 'grace_hash' -%} 0 \N 0 1 1 1 1 val2 0 \N 0 1 1 1 1 val7 0 \N 0 1 1 2 1 val5 @@ -94,6 +99,7 @@ ALL RIGHT | bs = {{ block_size }} | copmosite key 0 \N 0 2 2 \N 1 val9 2 2 2 2 2 2 0 val4 2 2 2 2 2 2 0 val4 +{% endif -%} ANY INNER USING | bs = {{ block_size }} 4 0 0 5 0 0 @@ -131,6 +137,7 @@ ANY LEFT | bs = {{ block_size }} 13 13 val13 0 14 14 val9 0 ANY RIGHT | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 4 4 0 val10 5 5 0 val6 6 6 0 val8 @@ -143,6 +150,7 @@ ANY RIGHT | bs = {{ block_size }} 13 13 0 val9 14 14 0 val3 14 14 0 val7 +{% endif -%} ANY INNER | bs = {{ block_size }} | copmosite key 2 2 2 2 2 2 0 0 ANY LEFT | bs = {{ block_size }} | copmosite key @@ -162,6 +170,7 @@ ANY LEFT | bs = {{ block_size }} | copmosite key 2 2 2 2 2 2 val12 0 2 2 2 2 2 2 val9 0 ANY RIGHT | bs = {{ block_size }} | copmosite key +{% if join_algorithm != 'grace_hash' -%} 0 \N 0 1 1 1 1 val2 0 \N 0 1 1 1 1 val7 0 \N 0 1 1 2 1 val5 @@ -174,6 +183,7 @@ ANY RIGHT | bs = {{ block_size }} | copmosite key 0 \N 0 2 1 \N 1 val3 0 \N 0 2 2 \N 1 val9 2 2 2 2 2 2 0 val4 +{% endif -%} {% endfor -%} ALL INNER | join_use_nulls = 1 4 4 0 0 @@ -209,6 +219,7 @@ ALL LEFT | join_use_nulls = 1 14 14 val9 0 14 14 val9 0 ALL RIGHT | join_use_nulls = 1 +{% if join_algorithm != 'grace_hash' -%} 4 4 0 val10 5 5 0 val6 6 6 0 val8 @@ -222,6 +233,7 @@ ALL RIGHT | join_use_nulls = 1 13 13 0 val9 14 14 0 val3 14 14 0 val7 +{% endif -%} ALL INNER | join_use_nulls = 1 | copmosite key 2 2 2 2 2 2 0 0 2 2 2 2 2 2 0 0 @@ -242,6 +254,7 @@ ALL LEFT | join_use_nulls = 1 | copmosite key 2 2 2 2 2 2 val12 0 2 2 2 2 2 2 val9 0 ALL RIGHT | join_use_nulls = 1 | copmosite key +{% if join_algorithm != 'grace_hash' -%} 2 2 2 2 2 2 0 val4 2 2 2 2 2 2 0 val4 \N \N \N 1 1 1 \N val2 @@ -255,6 +268,7 @@ ALL RIGHT | join_use_nulls = 1 | copmosite key \N \N \N 2 1 2 \N val8 \N \N \N 2 1 \N \N val3 \N \N \N 2 2 \N \N val9 +{% endif -%} ANY INNER | join_use_nulls = 1 4 4 0 0 5 5 0 0 @@ -282,6 +296,7 @@ ANY LEFT | join_use_nulls = 1 13 13 val13 0 14 14 val9 0 ANY RIGHT | join_use_nulls = 1 +{% if join_algorithm != 'grace_hash' -%} 4 4 0 val10 5 5 0 val6 6 6 0 val8 @@ -294,6 +309,7 @@ ANY RIGHT | join_use_nulls = 1 13 13 0 val9 14 14 0 val3 14 14 0 val7 +{% endif -%} ANY INNER | join_use_nulls = 1 | copmosite key 2 2 2 2 2 2 0 0 ANY LEFT | join_use_nulls = 1 | copmosite key @@ -313,6 +329,7 @@ ANY LEFT | join_use_nulls = 1 | copmosite key 2 2 2 2 2 2 val12 0 2 2 2 2 2 2 val9 0 ANY RIGHT | join_use_nulls = 1 | copmosite key +{% if join_algorithm != 'grace_hash' -%} 2 2 2 2 2 2 0 val4 \N \N \N 1 1 1 \N val2 \N \N \N 1 1 1 \N val7 @@ -325,3 +342,5 @@ ANY RIGHT | join_use_nulls = 1 | copmosite key \N \N \N 2 1 2 \N val8 \N \N \N 2 1 \N \N val3 \N \N \N 2 2 \N \N val9 +{% endif -%} +{% endfor -%} diff --git a/tests/queries/0_stateless/02273_full_sort_join.sql.j2 b/tests/queries/0_stateless/02273_full_sort_join.sql.j2 index b70d1e5f55f..8b739330364 100644 --- a/tests/queries/0_stateless/02273_full_sort_join.sql.j2 +++ b/tests/queries/0_stateless/02273_full_sort_join.sql.j2 @@ -26,7 +26,17 @@ INSERT INTO t2 'val' || toString(number) as s FROM numbers_mt({{ table_size - 3 }}); -SET join_algorithm = 'full_sorting_merge'; + +{% macro is_implemented(join_algorithm) -%} +{% if join_algorithm == 'grace_hash' %} -- { serverError NOT_IMPLEMENTED } {% endif %} +{% endmacro -%} + +{% for join_algorithm in ['default', 'full_sorting_merge', 'grace_hash'] -%} + +SET max_bytes_in_join = '{% if join_algorithm == 'grace_hash' %}10K{% else %}0{% endif %}'; + +SELECT '-- {{ join_algorithm }} --'; +SET join_algorithm = '{{ join_algorithm }}'; {% for block_size in range(1, table_size + 1) -%} {% for kind in ['ALL', 'ANY'] -%} @@ -59,7 +69,7 @@ SELECT t1.key, t2.key, empty(t1.s), t2.s FROM t1 {{ kind }} RIGHT JOIN t2 ON t1.key == t2.key ORDER BY t1.key, t2.key, t2.s -; +; {{ is_implemented(join_algorithm) }} SELECT '{{ kind }} INNER | bs = {{ block_size }} | copmosite key'; SELECT t1.key1, t1.key2, t1.key3, t2.key1, t2.key2, t2.key3, empty(t1.s), empty(t2.s) FROM t1 @@ -80,7 +90,7 @@ SELECT t1.key1, t1.key2, t1.key3, t2.key1, t2.key2, t2.key3, empty(t1.s), t2.s F {{ kind }} RIGHT JOIN t2 ON t1.key1 == t2.key1 AND t1.key2 == t2.key2 AND t1.key3 == t2.key3 AND t1.key1 == t2.key3 ORDER BY t1.key1, t1.key2, t1.key3, t2.key1, t2.key2, t2.key3, t2.s -; +; {{ is_implemented(join_algorithm) }} {% endfor -%} {% endfor -%} @@ -108,7 +118,7 @@ SELECT t1.key, t2.key, isNull(t1.s), t2.s FROM t1 {{ kind }} RIGHT JOIN t2 ON t1.key == t2.key ORDER BY t1.key, t2.key, t2.s -; +; {{ is_implemented(join_algorithm) }} SELECT '{{ kind }} INNER | join_use_nulls = 1 | copmosite key'; SELECT t1.key1, t1.key2, t1.key3, t2.key1, t2.key2, t2.key3, empty(t1.s), empty(t2.s) FROM t1 @@ -129,8 +139,12 @@ SELECT t1.key1, t1.key2, t1.key3, t2.key1, t2.key2, t2.key3, empty(t1.s), t2.s F {{ kind }} RIGHT JOIN t2 ON t1.key1 == t2.key1 AND t1.key2 == t2.key2 AND t1.key3 == t2.key3 AND t1.key1 == t2.key3 ORDER BY t1.key1, t1.key2, t1.key3, t2.key1, t2.key2, t2.key3, t2.s -; +; {{ is_implemented(join_algorithm) }} +SET join_use_nulls = 0; +SET max_bytes_in_join = 0; + +{% endfor -%} {% endfor -%} DROP TABLE IF EXISTS t1; diff --git a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2 b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2 index ca2e47d7208..2cc6c6e85d6 100644 --- a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2 +++ b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.reference.j2 @@ -1,3 +1,5 @@ +{% for join_algorithm in ['full_sorting_merge', 'grace_hash'] -%} +--- {{ join_algorithm }} --- {% for block_size in range(1, 11) -%} t1 ALL INNER JOIN t2 | bs = {{ block_size }} 1 1 4 5 @@ -106,6 +108,7 @@ t1 ALL LEFT JOIN t2 | bs = {{ block_size }} 2 2 val27 5 3 3 val3 4 t1 ALL RIGHT JOIN t2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 1 4 val11 1 1 4 val12 2 2 5 val22 @@ -158,6 +161,7 @@ t1 ALL RIGHT JOIN t2 | bs = {{ block_size }} 2 2 5 val28 2 2 5 val28 3 3 4 val3 +{% endif -%} t1 ANY INNER JOIN t2 | bs = {{ block_size }} 1 1 4 5 2 2 5 5 @@ -173,6 +177,7 @@ t1 ANY LEFT JOIN t2 | bs = {{ block_size }} 2 2 val27 5 3 3 val3 4 t1 ANY RIGHT JOIN t2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 1 4 val11 1 1 4 val12 2 2 5 val22 @@ -183,7 +188,9 @@ t1 ANY RIGHT JOIN t2 | bs = {{ block_size }} 2 2 5 val27 2 2 5 val28 3 3 4 val3 +{% endif -%} t1 ALL FULL JOIN t2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 1 4 5 1 1 4 5 2 2 5 5 @@ -236,7 +243,9 @@ t1 ALL FULL JOIN t2 | bs = {{ block_size }} 2 2 5 5 2 2 5 5 3 3 4 4 +{% endif -%} t1 ALL FULL JOIN USING t2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 4 5 1 4 5 2 5 5 @@ -289,6 +298,7 @@ t1 ALL FULL JOIN USING t2 | bs = {{ block_size }} 2 5 5 2 5 5 3 4 4 +{% endif -%} t1 ALL INNER JOIN tn2 | bs = {{ block_size }} 1 1 4 5 1 1 4 5 @@ -305,6 +315,7 @@ t1 ALL LEFT JOIN tn2 | bs = {{ block_size }} 2 \N val27 0 3 3 val3 4 t1 ALL RIGHT JOIN tn2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 0 \N 0 val22 0 \N 0 val23 0 \N 0 val24 @@ -315,6 +326,7 @@ t1 ALL RIGHT JOIN tn2 | bs = {{ block_size }} 1 1 4 val11 1 1 4 val12 3 3 4 val3 +{% endif -%} t1 ANY INNER JOIN tn2 | bs = {{ block_size }} 1 1 4 5 3 3 4 4 @@ -329,6 +341,7 @@ t1 ANY LEFT JOIN tn2 | bs = {{ block_size }} 2 \N val27 0 3 3 val3 4 t1 ANY RIGHT JOIN tn2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 0 \N 0 val22 0 \N 0 val23 0 \N 0 val24 @@ -339,7 +352,9 @@ t1 ANY RIGHT JOIN tn2 | bs = {{ block_size }} 1 1 4 val11 1 1 4 val12 3 3 4 val3 +{% endif -%} t1 ALL FULL JOIN tn2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 0 \N 0 5 0 \N 0 5 0 \N 0 5 @@ -357,7 +372,9 @@ t1 ALL FULL JOIN tn2 | bs = {{ block_size }} 2 \N 5 0 2 \N 5 0 3 3 4 4 +{% endif -%} t1 ALL FULL JOIN USING tn2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 4 5 1 4 5 2 5 0 @@ -375,6 +392,7 @@ t1 ALL FULL JOIN USING tn2 | bs = {{ block_size }} \N 0 5 \N 0 5 \N 0 5 +{% endif -%} tn1 ALL INNER JOIN t2 | bs = {{ block_size }} 1 1 4 5 1 1 4 5 @@ -391,6 +409,7 @@ tn1 ALL LEFT JOIN t2 | bs = {{ block_size }} \N 0 val26 0 \N 0 val27 0 tn1 ALL RIGHT JOIN t2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 1 4 val11 1 1 4 val12 3 3 4 val3 @@ -401,6 +420,7 @@ tn1 ALL RIGHT JOIN t2 | bs = {{ block_size }} \N 2 0 val26 \N 2 0 val27 \N 2 0 val28 +{% endif -%} tn1 ANY INNER JOIN t2 | bs = {{ block_size }} 1 1 4 5 3 3 4 4 @@ -415,6 +435,7 @@ tn1 ANY LEFT JOIN t2 | bs = {{ block_size }} \N 0 val26 0 \N 0 val27 0 tn1 ANY RIGHT JOIN t2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 1 4 val11 1 1 4 val12 3 3 4 val3 @@ -425,7 +446,9 @@ tn1 ANY RIGHT JOIN t2 | bs = {{ block_size }} \N 2 0 val26 \N 2 0 val27 \N 2 0 val28 +{% endif -%} tn1 ALL FULL JOIN t2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 1 4 5 1 1 4 5 3 3 4 4 @@ -443,7 +466,9 @@ tn1 ALL FULL JOIN t2 | bs = {{ block_size }} \N 2 0 5 \N 2 0 5 \N 2 0 5 +{% endif -%} tn1 ALL FULL JOIN USING t2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 4 5 1 4 5 2 0 5 @@ -461,6 +486,7 @@ tn1 ALL FULL JOIN USING t2 | bs = {{ block_size }} \N 5 0 \N 5 0 \N 5 0 +{% endif -%} tn1 ALL INNER JOIN tn2 | bs = {{ block_size }} 1 1 4 5 1 1 4 5 @@ -477,6 +503,7 @@ tn1 ALL LEFT JOIN tn2 | bs = {{ block_size }} \N \N val26 0 \N \N val27 0 tn1 ALL RIGHT JOIN tn2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 1 4 val11 1 1 4 val12 3 3 4 val3 @@ -487,6 +514,7 @@ tn1 ALL RIGHT JOIN tn2 | bs = {{ block_size }} \N \N 0 val26 \N \N 0 val27 \N \N 0 val28 +{% endif -%} tn1 ANY INNER JOIN tn2 | bs = {{ block_size }} 1 1 4 5 3 3 4 4 @@ -501,6 +529,7 @@ tn1 ANY LEFT JOIN tn2 | bs = {{ block_size }} \N \N val26 0 \N \N val27 0 tn1 ANY RIGHT JOIN tn2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 1 4 val11 1 1 4 val12 3 3 4 val3 @@ -511,7 +540,9 @@ tn1 ANY RIGHT JOIN tn2 | bs = {{ block_size }} \N \N 0 val26 \N \N 0 val27 \N \N 0 val28 +{% endif -%} tn1 ALL FULL JOIN tn2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 1 4 5 1 1 4 5 3 3 4 4 @@ -529,7 +560,9 @@ tn1 ALL FULL JOIN tn2 | bs = {{ block_size }} \N \N 5 0 \N \N 5 0 \N \N 5 0 +{% endif -%} tn1 ALL FULL JOIN USING tn2 | bs = {{ block_size }} +{% if join_algorithm != 'grace_hash' -%} 1 4 5 1 4 5 3 4 4 @@ -547,4 +580,6 @@ tn1 ALL FULL JOIN USING tn2 | bs = {{ block_size }} \N 5 0 \N 5 0 \N 5 0 +{% endif -%} +{% endfor -%} {% endfor -%} diff --git a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2 b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2 index 95d3a564016..613da65421e 100644 --- a/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2 +++ b/tests/queries/0_stateless/02274_full_sort_join_nodistinct.sql.j2 @@ -15,7 +15,17 @@ INSERT INTO tn1 VALUES (1, 'val1'), (NULL, 'val21'), (NULL, 'val22'), (NULL, 'va INSERT INTO t2 VALUES (1, 'val11'), (1, 'val12'), (2, 'val22'), (2, 'val23'), (2, 'val24'), (2, 'val25'), (2, 'val26'), (2, 'val27'), (2, 'val28'), (3, 'val3'); INSERT INTO tn2 VALUES (1, 'val11'), (1, 'val12'), (NULL, 'val22'), (NULL, 'val23'), (NULL, 'val24'), (NULL, 'val25'), (NULL, 'val26'), (NULL, 'val27'), (NULL, 'val28'), (3, 'val3'); -SET join_algorithm = 'full_sorting_merge'; +{% macro is_implemented(join_algorithm) -%} +{% if join_algorithm == 'grace_hash' %} -- { serverError NOT_IMPLEMENTED } {% endif %} +{% endmacro -%} + +{% for join_algorithm in ['full_sorting_merge', 'grace_hash'] -%} + +SET max_bytes_in_join = '{% if join_algorithm == 'grace_hash' %}10K{% else %}0{% endif %}'; + +SET join_algorithm = '{{ join_algorithm }}'; + +SELECT '--- {{ join_algorithm }} ---'; {% for block_size in range(1, 11) -%} SET max_block_size = {{ block_size }}; @@ -30,17 +40,20 @@ SELECT '{{ t1 }} {{ kind }} LEFT JOIN {{ t2 }} | bs = {{ block_size }}'; SELECT t1.key, t2.key, t1.s, length(t2.s) FROM {{ t1 }} AS t1 {{ kind }} LEFT JOIN {{ t2 }} AS t2 ON t1.key == t2.key ORDER BY t1.key, t2.key, t1.s; SELECT '{{ t1 }} {{ kind }} RIGHT JOIN {{ t2 }} | bs = {{ block_size }}'; -SELECT t1.key, t2.key, length(t1.s), t2.s FROM {{ t1 }} AS t1 {{ kind }} RIGHT JOIN {{ t2 }} AS t2 ON t1.key == t2.key ORDER BY t1.key, t2.key, t2.s; +SELECT t1.key, t2.key, length(t1.s), t2.s FROM {{ t1 }} AS t1 {{ kind }} RIGHT JOIN {{ t2 }} AS t2 ON t1.key == t2.key ORDER BY t1.key, t2.key, t2.s; {{ is_implemented(join_algorithm) }} {% endfor -%} SELECT '{{ t1 }} ALL FULL JOIN {{ t2 }} | bs = {{ block_size }}'; -SELECT t1.key, t2.key, length(t1.s), length(t2.s) FROM {{ t1 }} AS t1 {{ kind }} FULL JOIN {{ t2 }} AS t2 ON t1.key == t2.key ORDER BY t1.key, t2.key, length(t1.s), length(t2.s); +SELECT t1.key, t2.key, length(t1.s), length(t2.s) FROM {{ t1 }} AS t1 {{ kind }} FULL JOIN {{ t2 }} AS t2 ON t1.key == t2.key ORDER BY t1.key, t2.key, length(t1.s), length(t2.s); {{ is_implemented(join_algorithm) }} SELECT '{{ t1 }} ALL FULL JOIN USING {{ t2 }} | bs = {{ block_size }}'; -SELECT key, length(t1.s), length(t2.s) FROM {{ t1 }} AS t1 ALL FULL JOIN {{ t2 }} AS t2 USING (key) ORDER BY key, length(t1.s), length(t2.s); +SELECT key, length(t1.s), length(t2.s) FROM {{ t1 }} AS t1 ALL FULL JOIN {{ t2 }} AS t2 USING (key) ORDER BY key, length(t1.s), length(t2.s); {{ is_implemented(join_algorithm) }} {% endfor -%} +{% endfor -%} +SET max_bytes_in_join = 0; + {% endfor -%} DROP TABLE IF EXISTS t1; diff --git a/tests/queries/0_stateless/02275_full_sort_join_long.reference b/tests/queries/0_stateless/02275_full_sort_join_long.reference index 91b81d5ab3a..9ec06aea3e6 100644 --- a/tests/queries/0_stateless/02275_full_sort_join_long.reference +++ b/tests/queries/0_stateless/02275_full_sort_join_long.reference @@ -1,9 +1,4 @@ -ALL INNER -500353531835 500353531835 1000342 1000342 1000342 -ALL LEFT -50195752660639 500353531835 10369589 10369589 1000342 -ALL RIGHT -500353531835 684008812186 1367170 1000342 1367170 +-- full_sorting_merge -- ALL INNER 500353531835 500353531835 1000342 1000342 1000342 ALL LEFT @@ -40,9 +35,22 @@ ANY LEFT 50010619420459 315220291655 10000000 10000000 630753 ANY RIGHT 316611844056 500267124407 1000000 633172 1000000 -ANY INNER -199622811843 199622811843 399458 399458 399458 -ANY LEFT -50010619420459 315220291655 10000000 10000000 630753 -ANY RIGHT -316611844056 500267124407 1000000 633172 1000000 +-- grace_hash -- +ALL INNER +500353531835 500353531835 1000342 1000342 1000342 +ALL LEFT +50195752660639 500353531835 10369589 10369589 1000342 +ALL RIGHT +skipped +ALL INNER +500353531835 500353531835 1000342 1000342 1000342 +ALL LEFT +50195752660639 500353531835 10369589 10369589 1000342 +ALL RIGHT +skipped +ALL INNER +500353531835 500353531835 1000342 1000342 1000342 +ALL LEFT +50195752660639 500353531835 10369589 10369589 1000342 +ALL RIGHT +skipped diff --git a/tests/queries/0_stateless/02275_full_sort_join_long.sql.j2 b/tests/queries/0_stateless/02275_full_sort_join_long.sql.j2 index 29f1d46e2c8..98cc46c9cb4 100644 --- a/tests/queries/0_stateless/02275_full_sort_join_long.sql.j2 +++ b/tests/queries/0_stateless/02275_full_sort_join_long.sql.j2 @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-tsan, no-asan, no-ubsan, no-msan, no-debug DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; @@ -22,13 +22,26 @@ INSERT INTO t2 FROM numbers_mt({{ rtable_size }}) ; -SET join_algorithm = 'full_sorting_merge'; +{% macro is_implemented(join_algorithm) -%} +{% if join_algorithm == 'grace_hash' %} -- { serverError NOT_IMPLEMENTED } +SELECT 'skipped'; +{% endif -%} +{% endmacro -%} + +{% for join_algorithm in ['full_sorting_merge', 'grace_hash'] -%} + +SET max_bytes_in_join = '{% if join_algorithm == 'grace_hash' %}1M{% else %}0{% endif %}'; + +SELECT '-- {{ join_algorithm }} --'; +SET join_algorithm = '{{ join_algorithm }}'; {% for kind in ['ALL', 'ANY'] -%} -{% for block_size in [32001, 65505, 65536, range(32001, 65536) | random] %} +{% for block_size in [10240, 32001, 65536] %} SET max_block_size = {{ block_size }}; +{% if not (kind == 'ANY' and join_algorithm == 'grace_hash') -%} + SELECT '{{ kind }} INNER'; SELECT sum(t1.key), sum(t2.key), count(), countIf(t1.key != 0), countIf(t2.key != 0) FROM t1 {{ kind }} INNER JOIN t2 @@ -45,7 +58,13 @@ SELECT '{{ kind }} RIGHT'; SELECT sum(t1.key), sum(t2.key), count(), countIf(t1.key != 0), countIf(t2.key != 0) FROM t1 {{ kind }} RIGHT JOIN t2 ON t1.key == t2.key -; +; {{ is_implemented(join_algorithm) }} + +{% endif -%} {% endfor -%} {% endfor -%} + +SET max_bytes_in_join = 0; + +{% endfor -%} diff --git a/tests/queries/0_stateless/02293_part_log_has_merge_reason.sh b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sh index 1a33e6db459..23c073d2f83 100755 --- a/tests/queries/0_stateless/02293_part_log_has_merge_reason.sh +++ b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sh @@ -17,7 +17,7 @@ ${CLICKHOUSE_CLIENT} -q ' ENGINE = MergeTree() ORDER BY tuple() TTL event_time + INTERVAL 3 MONTH - SETTINGS min_bytes_for_wide_part = 0, materialize_ttl_recalculate_only = true, max_number_of_merges_with_ttl_in_pool = 100 + SETTINGS old_parts_lifetime = 1, min_bytes_for_wide_part = 0, materialize_ttl_recalculate_only = true, max_number_of_merges_with_ttl_in_pool = 100 ' ${CLICKHOUSE_CLIENT} -q "INSERT INTO t_part_log_has_merge_type_table VALUES (now(), 1, 'username1');" @@ -57,7 +57,7 @@ function wait_table_parts_are_merged_into_one_part() { export -f get_parts_count export -f wait_table_parts_are_merged_into_one_part -timeout 30 bash -c 'wait_table_parts_are_merged_into_one_part t_part_log_has_merge_type_table' +timeout 60 bash -c 'wait_table_parts_are_merged_into_one_part t_part_log_has_merge_type_table' ${CLICKHOUSE_CLIENT} -q 'SYSTEM FLUSH LOGS' diff --git a/tests/queries/0_stateless/02354_annoy.reference b/tests/queries/0_stateless/02354_annoy.reference index 2cc62ef4c86..38678fb67c9 100644 --- a/tests/queries/0_stateless/02354_annoy.reference +++ b/tests/queries/0_stateless/02354_annoy.reference @@ -14,3 +14,13 @@ 1 [0,0,10] 5 [0,0,10.2] 4 [0,0,9.7] + Name: annoy_index + Name: annoy_index +1 [0,0,10] +2 [0.2,0,10] +3 [-0.3,0,10] +1 [0,0,10] +2 [0.2,0,10] +3 [-0.3,0,10] + Name: annoy_index + Name: annoy_index diff --git a/tests/queries/0_stateless/02354_annoy.sh b/tests/queries/0_stateless/02354_annoy.sh new file mode 100755 index 00000000000..526886ec68d --- /dev/null +++ b/tests/queries/0_stateless/02354_annoy.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-ubsan, no-cpu-aarch64, no-backward-compatibility-check + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Check that index works correctly for L2Distance and with client parameters +$CLICKHOUSE_CLIENT -nm --allow_experimental_annoy_index=1 -q " +DROP TABLE IF EXISTS 02354_annoy_l2; + +CREATE TABLE 02354_annoy_l2 +( + id Int32, + embedding Array(Float32), + INDEX annoy_index embedding TYPE annoy() GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity=5; + +INSERT INTO 02354_annoy_l2 VALUES (1, [0.0, 0.0, 10.0]), (2, [0.0, 0.0, 10.5]), (3, [0.0, 0.0, 9.5]), (4, [0.0, 0.0, 9.7]), (5, [0.0, 0.0, 10.2]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]); + +SELECT * +FROM 02354_annoy_l2 +WHERE L2Distance(embedding, [0.0, 0.0, 10.0]) < 1.0 +LIMIT 5; + +SELECT * +FROM 02354_annoy_l2 +ORDER BY L2Distance(embedding, [0.0, 0.0, 10.0]) +LIMIT 3; + +SET param_02354_target_vector='[0.0, 0.0, 10.0]'; + +SELECT * +FROM 02354_annoy_l2 +WHERE L2Distance(embedding, {02354_target_vector: Array(Float32)}) < 1.0 +LIMIT 5; + +SELECT * +FROM 02354_annoy_l2 +ORDER BY L2Distance(embedding, {02354_target_vector: Array(Float32)}) +LIMIT 3; + +SELECT * +FROM 02354_annoy_l2 +ORDER BY L2Distance(embedding, [0.0, 0.0]) +LIMIT 3; -- { serverError 80 } + + +DROP TABLE IF EXISTS 02354_annoy_l2; +" + +# Check that indexes are used +$CLICKHOUSE_CLIENT -nm --allow_experimental_annoy_index=1 -q " +DROP TABLE IF EXISTS 02354_annoy_l2; + +CREATE TABLE 02354_annoy_l2 +( + id Int32, + embedding Array(Float32), + INDEX annoy_index embedding TYPE annoy() GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity=5; + +INSERT INTO 02354_annoy_l2 VALUES (1, [0.0, 0.0, 10.0]), (2, [0.0, 0.0, 10.5]), (3, [0.0, 0.0, 9.5]), (4, [0.0, 0.0, 9.7]), (5, [0.0, 0.0, 10.2]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]); + +EXPLAIN indexes=1 +SELECT * +FROM 02354_annoy_l2 +WHERE L2Distance(embedding, [0.0, 0.0, 10.0]) < 1.0 +LIMIT 5; + +EXPLAIN indexes=1 +SELECT * +FROM 02354_annoy_l2 +ORDER BY L2Distance(embedding, [0.0, 0.0, 10.0]) +LIMIT 3; +DROP TABLE IF EXISTS 02354_annoy_l2; +" | grep "annoy_index" + + +# # Check that index works correctly for cosineDistance +$CLICKHOUSE_CLIENT -nm --allow_experimental_annoy_index=1 -q " +DROP TABLE IF EXISTS 02354_annoy_cosine; + +CREATE TABLE 02354_annoy_cosine +( + id Int32, + embedding Array(Float32), + INDEX annoy_index embedding TYPE annoy(100, 'cosineDistance') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity=5; + +INSERT INTO 02354_annoy_cosine VALUES (1, [0.0, 0.0, 10.0]), (2, [0.2, 0.0, 10.0]), (3, [-0.3, 0.0, 10.0]), (4, [0.5, 0.0, 10.1]), (5, [0.8, 0.0, 10.0]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]); + +SELECT * +FROM 02354_annoy_cosine +WHERE cosineDistance(embedding, [0.0, 0.0, 10.0]) < 1.0 +LIMIT 3; + +SELECT * +FROM 02354_annoy_cosine +ORDER BY cosineDistance(embedding, [0.0, 0.0, 10.0]) +LIMIT 3; + +DROP TABLE IF EXISTS 02354_annoy_cosine; +" + +# # Check that indexes are used +$CLICKHOUSE_CLIENT -nm --allow_experimental_annoy_index=1 -q " +DROP TABLE IF EXISTS 02354_annoy_cosine; + +CREATE TABLE 02354_annoy_cosine +( + id Int32, + embedding Array(Float32), + INDEX annoy_index embedding TYPE annoy(100, 'cosineDistance') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity=5; + +INSERT INTO 02354_annoy_cosine VALUES (1, [0.0, 0.0, 10.0]), (2, [0.2, 0.0, 10.0]), (3, [-0.3, 0.0, 10.0]), (4, [0.5, 0.0, 10.1]), (5, [0.8, 0.0, 10.0]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]); + +EXPLAIN indexes=1 +SELECT * +FROM 02354_annoy_cosine +WHERE cosineDistance(embedding, [0.0, 0.0, 10.0]) < 1.0 +LIMIT 3; + +EXPLAIN indexes=1 +SELECT * +FROM 02354_annoy_cosine +ORDER BY cosineDistance(embedding, [0.0, 0.0, 10.0]) +LIMIT 3; +DROP TABLE IF EXISTS 02354_annoy_cosine; +" | grep "annoy_index" + +# # Check that weird base columns are rejected +$CLICKHOUSE_CLIENT -nm --allow_experimental_annoy_index=1 -q " +DROP TABLE IF EXISTS 02354_annoy; + +-- Index spans >1 column + +CREATE TABLE 02354_annoy +( + id Int32, + embedding Array(Float32), + INDEX annoy_index (embedding, id) TYPE annoy(100) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity=5; -- {serverError 7 } + +-- Index must be created on Array(Float32) or Tuple(Float32) + +CREATE TABLE 02354_annoy +( + id Int32, + embedding Float32, + INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity=5; -- {serverError 44 } + + +CREATE TABLE 02354_annoy +( + id Int32, + embedding Array(Float64), + INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity=5; -- {serverError 44 } + +CREATE TABLE 02354_annoy +( + id Int32, + embedding Tuple(Float32, Float64), + INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity=5; -- {serverError 44 } + +CREATE TABLE 02354_annoy +( + id Int32, + embedding Array(LowCardinality(Float32)), + INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity=5; -- {serverError 44 } + +CREATE TABLE 02354_annoy +( + id Int32, + embedding Array(Nullable(Float32)), + INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity=5; -- {serverError 44 }" diff --git a/tests/queries/0_stateless/02354_annoy.sql b/tests/queries/0_stateless/02354_annoy.sql deleted file mode 100644 index 654a4b545ea..00000000000 --- a/tests/queries/0_stateless/02354_annoy.sql +++ /dev/null @@ -1,114 +0,0 @@ --- Tags: no-fasttest, no-ubsan, no-cpu-aarch64, no-backward-compatibility-check - -SET allow_experimental_annoy_index = 1; - -DROP TABLE IF EXISTS 02354_annoy; - -CREATE TABLE 02354_annoy -( - id Int32, - embedding Array(Float32), - INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 -) -ENGINE = MergeTree -ORDER BY id -SETTINGS index_granularity=5; - -INSERT INTO 02354_annoy VALUES (1, [0.0, 0.0, 10.0]), (2, [0.0, 0.0, 10.5]), (3, [0.0, 0.0, 9.5]), (4, [0.0, 0.0, 9.7]), (5, [0.0, 0.0, 10.2]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]); - -SELECT * -FROM 02354_annoy -WHERE L2Distance(embedding, [0.0, 0.0, 10.0]) < 1.0 -LIMIT 5; - -SELECT * -FROM 02354_annoy -ORDER BY L2Distance(embedding, [0.0, 0.0, 10.0]) -LIMIT 3; - -SET param_02354_target_vector='[0.0, 0.0, 10.0]'; - -SELECT * -FROM 02354_annoy -WHERE L2Distance(embedding, {02354_target_vector: Array(Float32)}) < 1.0 -LIMIT 5; - -SELECT * -FROM 02354_annoy -ORDER BY L2Distance(embedding, {02354_target_vector: Array(Float32)}) -LIMIT 3; - -SELECT * -FROM 02354_annoy -ORDER BY L2Distance(embedding, [0.0, 0.0]) -LIMIT 3; -- { serverError 80 } - -DROP TABLE IF EXISTS 02354_annoy; - --- ------------------------------------ --- Check that weird base columns are rejected - --- Index spans >1 column - -CREATE TABLE 02354_annoy -( - id Int32, - embedding Array(Float32), - INDEX annoy_index (embedding, id) TYPE annoy(100) GRANULARITY 1 -) -ENGINE = MergeTree -ORDER BY id -SETTINGS index_granularity=5; -- {serverError 7 } - --- Index must be created on Array(Float32) or Tuple(Float32) - -CREATE TABLE 02354_annoy -( - id Int32, - embedding Float32, - INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 -) -ENGINE = MergeTree -ORDER BY id -SETTINGS index_granularity=5; -- {serverError 44 } - - -CREATE TABLE 02354_annoy -( - id Int32, - embedding Array(Float64), - INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 -) -ENGINE = MergeTree -ORDER BY id -SETTINGS index_granularity=5; -- {serverError 44 } - -CREATE TABLE 02354_annoy -( - id Int32, - embedding Tuple(Float32, Float64), - INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 -) -ENGINE = MergeTree -ORDER BY id -SETTINGS index_granularity=5; -- {serverError 44 } - -CREATE TABLE 02354_annoy -( - id Int32, - embedding Array(LowCardinality(Float32)), - INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 -) -ENGINE = MergeTree -ORDER BY id -SETTINGS index_granularity=5; -- {serverError 44 } - -CREATE TABLE 02354_annoy -( - id Int32, - embedding Array(Nullable(Float32)), - INDEX annoy_index embedding TYPE annoy(100) GRANULARITY 1 -) -ENGINE = MergeTree -ORDER BY id -SETTINGS index_granularity=5; -- {serverError 44 } diff --git a/tests/queries/0_stateless/02367_join_pushdown_column_not_found.reference b/tests/queries/0_stateless/02367_join_pushdown_column_not_found.reference index 98fb6a68656..627e1097cda 100644 --- a/tests/queries/0_stateless/02367_join_pushdown_column_not_found.reference +++ b/tests/queries/0_stateless/02367_join_pushdown_column_not_found.reference @@ -2,3 +2,4 @@ 1 1 1 +1 diff --git a/tests/queries/0_stateless/02367_join_pushdown_column_not_found.sql.j2 b/tests/queries/0_stateless/02367_join_pushdown_column_not_found.sql.j2 index 95f3c5be711..86e7bca00a9 100644 --- a/tests/queries/0_stateless/02367_join_pushdown_column_not_found.sql.j2 +++ b/tests/queries/0_stateless/02367_join_pushdown_column_not_found.sql.j2 @@ -1,4 +1,4 @@ -{% for join_algorithm in ['default', 'full_sorting_merge', 'hash', 'partial_merge'] -%} +{% for join_algorithm in ['default', 'full_sorting_merge', 'hash', 'partial_merge', 'grace_hash'] -%} SET join_algorithm = '{{ join_algorithm }}'; diff --git a/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql b/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql index cf5ca15adeb..0f1b4f638cb 100644 --- a/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql +++ b/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql @@ -1,7 +1,7 @@ -- Tags: no-backward-compatibility-check drop table if exists test_02381; -create table test_02381(a UInt64, b UInt64) ENGINE = MergeTree order by (a, b); +create table test_02381(a UInt64, b UInt64) ENGINE = MergeTree order by (a, b) SETTINGS compress_marks=false, compress_primary_key=false; insert into test_02381 select number, number * 10 from system.numbers limit 1000000; drop table if exists test_02381_compress; diff --git a/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.reference b/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.reference index 7f79a172f4b..083f0f69dc8 100644 --- a/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.reference +++ b/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.reference @@ -3,7 +3,7 @@ SET enable_filesystem_cache_on_write_operations=0; SYSTEM DROP FILESYSTEM CACHE; DROP TABLE IF EXISTS nopers; -CREATE TABLE nopers (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760; +CREATE TABLE nopers (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES nopers; INSERT INTO nopers SELECT number, toString(number) FROM numbers(10); SELECT * FROM nopers FORMAT Null; @@ -22,7 +22,7 @@ ORDER BY file, cache, size; data.bin 0 114 data.mrk3 0 80 DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; INSERT INTO test SELECT number, toString(number) FROM numbers(100); SELECT * FROM test FORMAT Null; @@ -47,7 +47,7 @@ data.bin 0 746 data.mrk3 0 80 data.mrk3 0_persistent 80 DROP TABLE IF EXISTS test2; -CREATE TABLE test2 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760; +CREATE TABLE test2 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test2; INSERT INTO test2 SELECT number, toString(number) FROM numbers(100000); SELECT * FROM test2 FORMAT Null; diff --git a/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.sql b/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.sql index d7171de48ad..6486840602e 100644 --- a/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.sql +++ b/tests/queries/0_stateless/02382_filesystem_cache_persistent_files.sql @@ -7,7 +7,7 @@ SET enable_filesystem_cache_on_write_operations=0; SYSTEM DROP FILESYSTEM CACHE; DROP TABLE IF EXISTS nopers; -CREATE TABLE nopers (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760; +CREATE TABLE nopers (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES nopers; INSERT INTO nopers SELECT number, toString(number) FROM numbers(10); @@ -26,7 +26,7 @@ ON data_paths.cache_path = caches.cache_path ORDER BY file, cache, size; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; INSERT INTO test SELECT number, toString(number) FROM numbers(100); @@ -49,7 +49,7 @@ ON data_paths.cache_path = caches.cache_path ORDER BY file, cache, size; DROP TABLE IF EXISTS test2; -CREATE TABLE test2 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760; +CREATE TABLE test2 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test2; INSERT INTO test2 SELECT number, toString(number) FROM numbers(100000); diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.reference b/tests/queries/0_stateless/02404_memory_bound_merging.reference new file mode 100644 index 00000000000..47d3470ef6e --- /dev/null +++ b/tests/queries/0_stateless/02404_memory_bound_merging.reference @@ -0,0 +1,141 @@ +-- { echoOn } -- +explain pipeline select a from remote(test_cluster_two_shards, currentDatabase(), t) group by a; +(Expression) +ExpressionTransform × 4 + (MergingAggregated) + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 2 → 1 + (Union) + (Aggregating) + SortingAggregatedForMemoryBoundMergingTransform 4 → 1 + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 4 → 1 + AggregatingInOrderTransform × 4 + (Expression) + ExpressionTransform × 4 + (ReadFromMergeTree) + MergeTreeInOrder × 4 0 → 1 + (ReadFromRemote) +select a from remote(test_cluster_two_shards, currentDatabase(), t) group by a order by a limit 5 offset 100500; +100500 +100501 +100502 +100503 +100504 +explain pipeline select a from remote(test_cluster_two_shards, currentDatabase(), dist_t) group by a; +(Expression) +ExpressionTransform × 4 + (MergingAggregated) + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 2 → 1 + (Union) + (MergingAggregated) + SortingAggregatedForMemoryBoundMergingTransform 4 → 1 + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 2 → 1 + (Union) + (Aggregating) + SortingAggregatedForMemoryBoundMergingTransform 4 → 1 + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 4 → 1 + AggregatingInOrderTransform × 4 + (Expression) + ExpressionTransform × 4 + (ReadFromMergeTree) + MergeTreeInOrder × 4 0 → 1 + (ReadFromRemote) + (ReadFromRemote) +select a from remote(test_cluster_two_shards, currentDatabase(), dist_t) group by a order by a limit 5 offset 100500; +100500 +100501 +100502 +100503 +100504 +1 +-- { echoOn } -- +explain pipeline select a, count() from dist_t_different_dbs group by a order by a limit 5 offset 500; +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + MergingSortedTransform 4 → 1 + MergeSortingTransform × 4 + LimitsCheckingTransform × 4 + PartialSortingTransform × 4 + (Expression) + ExpressionTransform × 4 + (MergingAggregated) + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 2 → 1 + (Union) + (Aggregating) + SortingAggregatedForMemoryBoundMergingTransform 4 → 1 + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 4 → 1 + AggregatingInOrderTransform × 4 + (Expression) + ExpressionTransform × 4 + (ReadFromMergeTree) + MergeTreeInOrder × 4 0 → 1 + (ReadFromRemote) +select a, count() from dist_t_different_dbs group by a order by a limit 5 offset 500; +500 2000 +501 2000 +502 2000 +503 2000 +504 2000 +select a, count() from dist_t_different_dbs group by a, b order by a limit 5 offset 500; +500 2000 +501 2000 +502 2000 +503 2000 +504 2000 +-- { echoOn } -- +explain pipeline select a from dist_pr_t group by a order by a limit 5 offset 500; +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + MergingSortedTransform 4 → 1 + MergeSortingTransform × 4 + LimitsCheckingTransform × 4 + PartialSortingTransform × 4 + (Expression) + ExpressionTransform × 4 + (MergingAggregated) + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 3 → 1 + (Union) + (Aggregating) + SortingAggregatedForMemoryBoundMergingTransform 4 → 1 + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 4 → 1 + AggregatingInOrderTransform × 4 + (Expression) + ExpressionTransform × 4 + (ReadFromMergeTree) + MergeTreeInOrder × 4 0 → 1 + (ReadFromRemoteParallelReplicas) +select a, count() from dist_pr_t group by a order by a limit 5 offset 500; +500 1000 +501 1000 +502 1000 +503 1000 +504 1000 +select a, count() from dist_pr_t group by a, b order by a limit 5 offset 500; +500 1000 +501 1000 +502 1000 +503 1000 +504 1000 diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.sql b/tests/queries/0_stateless/02404_memory_bound_merging.sql new file mode 100644 index 00000000000..c41e2d3abae --- /dev/null +++ b/tests/queries/0_stateless/02404_memory_bound_merging.sql @@ -0,0 +1,72 @@ +-- Tags: no-parallel + +create table t(a UInt64, b UInt64) engine=MergeTree order by a; +system stop merges t; +insert into t select number, number from numbers_mt(1e6); + +set enable_memory_bound_merging_of_aggregation_results = 1; +set max_threads = 4; +set optimize_aggregation_in_order = 1; +set prefer_localhost_replica = 1; + +-- slightly different transforms will be generated by reading steps if we let settings randomisation to change this setting value -- +set read_in_order_two_level_merge_threshold = 1000; + +create table dist_t as t engine = Distributed(test_cluster_two_shards, currentDatabase(), t, a % 2); + +-- { echoOn } -- +explain pipeline select a from remote(test_cluster_two_shards, currentDatabase(), t) group by a; + +select a from remote(test_cluster_two_shards, currentDatabase(), t) group by a order by a limit 5 offset 100500; + +explain pipeline select a from remote(test_cluster_two_shards, currentDatabase(), dist_t) group by a; + +select a from remote(test_cluster_two_shards, currentDatabase(), dist_t) group by a order by a limit 5 offset 100500; + +-- { echoOff } -- + +set aggregation_in_order_max_block_bytes = '1Mi'; +set max_block_size = 500; +-- actual block size might be slightly bigger than the limit -- +select max(bs) < 70000 from (select avg(a), max(blockSize()) as bs from remote(test_cluster_two_shards, currentDatabase(), t) group by a); + +-- beautiful case when we have different sorting key definitions in tables involved in distributed query => different plans => different sorting properties of local aggregation results -- +create database if not exists shard_1; +create table t_different_dbs(a UInt64, b UInt64) engine = MergeTree order by a; +create table shard_1.t_different_dbs(a UInt64, b UInt64) engine = MergeTree order by tuple(); + +insert into t_different_dbs select number % 1000, number % 1000 from numbers_mt(1e6); +insert into shard_1.t_different_dbs select number % 1000, number % 1000 from numbers_mt(1e6); + +create table dist_t_different_dbs as t engine = Distributed(test_cluster_two_shards_different_databases_with_local, '', t_different_dbs); + +-- { echoOn } -- +explain pipeline select a, count() from dist_t_different_dbs group by a order by a limit 5 offset 500; + +select a, count() from dist_t_different_dbs group by a order by a limit 5 offset 500; +select a, count() from dist_t_different_dbs group by a, b order by a limit 5 offset 500; + +-- { echoOff } -- + +set allow_experimental_parallel_reading_from_replicas = 1; +set max_parallel_replicas = 3; +set use_hedged_requests = 0; + +create table pr_t(a UInt64, b UInt64) engine=MergeTree order by a; +insert into pr_t select number % 1000, number % 1000 from numbers_mt(1e6); +create table dist_pr_t as pr_t engine = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), pr_t); + +-- { echoOn } -- +explain pipeline select a from dist_pr_t group by a order by a limit 5 offset 500; + +select a, count() from dist_pr_t group by a order by a limit 5 offset 500; +select a, count() from dist_pr_t group by a, b order by a limit 5 offset 500; + +-- { echoOff } -- + +drop table dist_pr_t; +drop table dist_t_different_dbs; +drop table shard_1.t_different_dbs; +drop table t_different_dbs; +drop table dist_t; +drop table t; diff --git a/tests/queries/0_stateless/02417_from_select_syntax.reference b/tests/queries/0_stateless/02417_from_select_syntax.reference new file mode 100644 index 00000000000..44e0be8e356 --- /dev/null +++ b/tests/queries/0_stateless/02417_from_select_syntax.reference @@ -0,0 +1,4 @@ +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/02417_from_select_syntax.sql b/tests/queries/0_stateless/02417_from_select_syntax.sql new file mode 100644 index 00000000000..ce6cb3a14da --- /dev/null +++ b/tests/queries/0_stateless/02417_from_select_syntax.sql @@ -0,0 +1,4 @@ +FROM numbers(1) SELECT number; +WITH 1 as n FROM numbers(1) SELECT number * n; +FROM (FROM numbers(1) SELECT *) SELECT number; +FROM (FROM numbers(1) SELECT *) AS select SELECT number; diff --git a/tests/queries/0_stateless/02421_truncate_isolation_no_merges.reference b/tests/queries/0_stateless/02421_truncate_isolation_no_merges.reference new file mode 100644 index 00000000000..a89ce339f6c --- /dev/null +++ b/tests/queries/0_stateless/02421_truncate_isolation_no_merges.reference @@ -0,0 +1,51 @@ +concurrent_drop_after +tx11 3 +concurrent_drop_before +tx21 3 +UNKNOWN_TABLE +concurrent_insert +2 +all_1_1_1 0 +all_2_2_1 0 +all_3_3_1 0 +all_4_4_1 0 +all_5_5_0 1 +all_6_6_1 0 +concurrent_drop_part_before +SERIALIZATION_ERROR +INVALID_TRANSACTION +1 +3 +all_1_1_0 1 +all_2_2_1 0 +all_3_3_0 1 +read_from_snapshot +tx51 3 +tx51 3 +tx52 0 +tx51 3 +0 +concurrent_drop_part_after +NO_SUCH_DATA_PART +INVALID_TRANSACTION +all_1_1_1 0 +all_2_2_1 0 +all_3_3_1 0 +NewPart all_1_1_0 +NewPart all_1_1_1 +NewPart all_2_2_0 +NewPart all_2_2_1 +NewPart all_3_3_0 +NewPart all_3_3_1 +concurrent_truncate_notx_after +tx71 3 +tx71 0 +0 +concurrent_truncate_notx_before +tx81 3 +NO_SUCH_DATA_PART +INVALID_TRANSACTION +INVALID_TRANSACTION +0 +concurrent_rollback_truncate +3 diff --git a/tests/queries/0_stateless/02421_truncate_isolation_no_merges.sh b/tests/queries/0_stateless/02421_truncate_isolation_no_merges.sh new file mode 100755 index 00000000000..b1e8500a4d4 --- /dev/null +++ b/tests/queries/0_stateless/02421_truncate_isolation_no_merges.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database, no-ordinary-database, long + +set -e -o pipefail + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh +# shellcheck source=./transactions.lib +. "$CURDIR"/transactions.lib + + +function reset_table() +{ + table=${1:-"tt"} + $CLICKHOUSE_CLIENT -q "drop table if exists $table" + $CLICKHOUSE_CLIENT -q "create table $table (n int) engine=MergeTree order by tuple()" + + # In order to preserve parts names merges have to be disabled + $CLICKHOUSE_CLIENT -q "system stop merges $table" + + $CLICKHOUSE_CLIENT -q "insert into $table values (1)" # inserts all_1_1_0 + $CLICKHOUSE_CLIENT -q "insert into $table values (2)" # inserts all_2_2_0 + $CLICKHOUSE_CLIENT -q "insert into $table values (3)" # inserts all_3_3_0 +} + +function concurrent_drop_after() +{ + echo "concurrent_drop_after" + + reset_table + + tx 11 "begin transaction" + tx 11 "select count() from tt" + tx 11 "truncate table tt" + $CLICKHOUSE_CLIENT --database_atomic_wait_for_drop_and_detach_synchronously=0 -q "drop table tt" + tx 11 "commit" +} + +concurrent_drop_after + +function concurrent_drop_before() +{ + echo "concurrent_drop_before" + + reset_table + + tx 21 "begin transaction" + tx 21 "select count() from tt" + $CLICKHOUSE_CLIENT -q "drop table tt" + tx 21 "truncate table tt" | grep -Eo "UNKNOWN_TABLE" | uniq + tx 21 "rollback" +} + +concurrent_drop_before + +function concurrent_insert() +{ + echo "concurrent_insert" + + reset_table + + tx 31 "begin transaction" + tx 32 "begin transaction" + tx 31 "insert into tt values (1)" # inserts all_4_4_0 + tx 32 "insert into tt values (2)" # inserts all_5_5_0 + tx 31 "insert into tt values (3)" # inserts all_6_6_0 + tx 31 "truncate table tt" # creates all_1_4_1 all_6_6_1 + tx 31 "commit" + tx 32 "commit" + + $CLICKHOUSE_CLIENT -q "select n from tt order by n" + $CLICKHOUSE_CLIENT -q "select name, rows from system.parts + where table='tt' and database=currentDatabase() and active + order by name" +} + +concurrent_insert + +function concurrent_drop_part_before() +{ + echo "concurrent_drop_part_before" + + reset_table + + tx 41 "begin transaction" + tx 42 "begin transaction" + tx 42 "alter table tt drop part 'all_2_2_0'" + tx 41 "truncate table tt" | grep -Eo "SERIALIZATION_ERROR" | uniq + tx 41 "commit" | grep -Eo "INVALID_TRANSACTION" | uniq + tx 42 "commit" + + $CLICKHOUSE_CLIENT -q "select n from tt order by n" + $CLICKHOUSE_CLIENT -q "select name, rows from system.parts + where table='tt' and database=currentDatabase() and active + order by name" + + reset_table +} + +concurrent_drop_part_before + +function read_from_snapshot() +{ + echo "read_from_snapshot" + + reset_table + + tx 51 "begin transaction" + tx 51 "select count() from tt" + tx 52 "begin transaction" + tx 52 "truncate table tt" + tx 51 "select count() from tt" + tx 52 "select count() from tt" + tx 52 "commit" + tx 51 "select count() from tt" + tx 51 "commit" + + $CLICKHOUSE_CLIENT -q "select count() from tt" +} + +read_from_snapshot + + +function concurrent_drop_part_after() +{ + echo "concurrent_drop_part_after" + + reset_table drop_part_after_table + + tx 61 "begin transaction" + tx 62 "begin transaction" + tx 61 "truncate table drop_part_after_table" + tx 62 "alter table drop_part_after_table drop part 'all_2_2_0'" | grep -Eo "NO_SUCH_DATA_PART" | uniq + tx 61 "commit" + tx 62 "commit" | grep -Eo "INVALID_TRANSACTION" | uniq + + $CLICKHOUSE_CLIENT -q "select n from drop_part_after_table order by n" + $CLICKHOUSE_CLIENT -q "select name, rows from system.parts + where table='drop_part_after_table' and database=currentDatabase() and active + order by name" + $CLICKHOUSE_CLIENT -q "system flush logs" + $CLICKHOUSE_CLIENT -q "select event_type, part_name from system.part_log + where table='drop_part_after_table' and database=currentDatabase() + order by part_name" +} + +concurrent_drop_part_after + +function concurrent_truncate_notx_after() +{ + echo "concurrent_truncate_notx_after" + + reset_table + + tx 71 "begin transaction" + tx 71 "select count() from tt" + tx 71 "alter table tt drop part 'all_2_2_0'" + $CLICKHOUSE_CLIENT -q "truncate table tt" + # return 0, since truncate was out of transaction + # it would be better if exception raised + tx 71 "select count() from tt" + tx 71 "commit" + + $CLICKHOUSE_CLIENT -q "select count() from tt" +} + +concurrent_truncate_notx_after + +function concurrent_truncate_notx_before() +{ + echo "concurrent_truncate_notx_before" + + reset_table + + tx 81 "begin transaction" + tx 81 "select count() from tt" + $CLICKHOUSE_CLIENT -q "truncate table tt" + tx 81 "alter table tt drop part 'all_2_2_0'" | grep -Eo "NO_SUCH_DATA_PART" | uniq + tx 81 "select count() from tt" | grep -Eo "INVALID_TRANSACTION" | uniq + tx 81 "commit" | grep -Eo "INVALID_TRANSACTION" | uniq + + $CLICKHOUSE_CLIENT -q "select count() from tt" +} + +concurrent_truncate_notx_before + +function concurrent_rollback_truncate() +{ + echo "concurrent_rollback_truncate" + + reset_table + + tx 91 "begin transaction" + tx 92 "begin transaction" + tx 91 "truncate table tt" + tx_async 91 "rollback" + tx 92 "truncate table tt" | grep -vwe "PART_IS_TEMPORARILY_LOCKED" -vwe "SERIALIZATION_ERROR" ||: + tx 92 "rollback" + tx_wait 91 + + $CLICKHOUSE_CLIENT -q "select count() from tt" +} + +concurrent_rollback_truncate diff --git a/tests/queries/0_stateless/02421_truncate_isolation_with_mutations.reference b/tests/queries/0_stateless/02421_truncate_isolation_with_mutations.reference new file mode 100644 index 00000000000..5890f1120db --- /dev/null +++ b/tests/queries/0_stateless/02421_truncate_isolation_with_mutations.reference @@ -0,0 +1,60 @@ +concurrent_delete_before +tx11 41 3 +tx11 41 3 +SERIALIZATION_ERROR +tx12 42 1 +2 +4 +concurrent_delete_after +tx21 111 3 +tx22 112 3 +UNFINISHED +concurrent_delete_rollback +tx31 3 +tx31 3 +tx32 1 +tx31 3 +0 +concurrent_optimize_table_not_start +tx41 4 +3 all_1_1_0 +1 all_2_2_0 +concurrent_optimize_table +tx43 5 +SERIALIZATION_ERROR +INVALID_TRANSACTION +5 all_1_2_1 +1 all_3_3_0 +concurrent_optimize_table_before +3 all_1_1_0 +drop_parts_which_already_outdated +tx69 before optimize 3 all_1_1_6 +tx69 before optimize 1 all_2_2_0 +tx69 after optimize 3 all_1_1_6 +tx69 after optimize 1 all_2_2_0 +SERIALIZATION_ERROR +at the end 4 all_1_2_7 +unable_drop_one_part_which_outdated_but_visible +tx79 before optimize 3 all_1_1_2 +tx79 before optimize 1 all_2_2_0 +tx79 after optimize 3 all_1_1_2 +tx79 after optimize 1 all_2_2_0 +NO_SUCH_DATA_PART +at the end 3 all_1_1_2 +at the end 1 all_2_2_0 +drop_one_part_which_outdated_and_reverted +tx89 before optimize 3 all_1_1_1 +tx89 before optimize 1 all_2_2_0 +tx89 after optimize 3 all_1_1_1 +tx89 after optimize 1 all_2_2_0 +tx89 after rollback 3 all_1_1_1 +tx89 after rollback 1 all_2_2_0 +at the end 3 all_1_1_1 +drop_one_part_which_outdated_and_reverted_no_name_intersection +tx99 before optimize 3 all_1_1_0 +tx99 before optimize 1 all_2_2_0 +tx99 after optimize 3 all_1_1_0 +tx99 after optimize 1 all_2_2_0 +tx99 after rollback 3 all_1_1_0 +tx99 after rollback 1 all_2_2_0 +at the end 3 all_1_1_0 diff --git a/tests/queries/0_stateless/02421_truncate_isolation_with_mutations.sh b/tests/queries/0_stateless/02421_truncate_isolation_with_mutations.sh new file mode 100755 index 00000000000..fabc9eab140 --- /dev/null +++ b/tests/queries/0_stateless/02421_truncate_isolation_with_mutations.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database, no-ordinary-database, long + +set -e -o pipefail + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh +# shellcheck source=./transactions.lib +. "$CURDIR"/transactions.lib +# shellcheck source=./parts.lib +. "$CURDIR"/parts.lib + +function reset_table() +{ + table=${1:-"tt"} + settings=${2:-""} + $CLICKHOUSE_CLIENT -q "drop table if exists $table" + $CLICKHOUSE_CLIENT -q "create table $table (n int) engine=MergeTree order by tuple() $settings" + + $CLICKHOUSE_CLIENT -q "insert into $table values (1), (2), (3)" # inserts all_1_1_0 +} + +function concurrent_delete_before() +{ + $CLICKHOUSE_CLIENT -q "select 'concurrent_delete_before'" + + reset_table tt + + tx 11 "begin transaction" + tx 11 "select 41, count() from tt" + tx 12 "begin transaction" + tx 12 "alter table tt delete where n%2=1" + tx 11 "select 41, count() from tt" + tx 11 "truncate table tt" | grep -Eo "SERIALIZATION_ERROR" | uniq + tx 12 "select 42, count() from tt" + tx 11 "rollback" + tx 12 "insert into tt values (4)" + tx 12 "commit" + + $CLICKHOUSE_CLIENT -q "select n from tt order by n" +} + +concurrent_delete_before + +function concurrent_delete_after() +{ + $CLICKHOUSE_CLIENT -q "select 'concurrent_delete_after'" + + reset_table tt + + tx 21 "begin transaction" + tx 22 "begin transaction" + tx 21 "select 111, count() from tt" + tx 21 "truncate table tt" + tx 22 "select 112, count() from tt" + tx 22 "alter table tt delete where n%2=1" | grep -Eo "UNFINISHED" | uniq + tx 21 "commit" + tx 22 "rollback" + + $CLICKHOUSE_CLIENT -q "select n from tt order by n" +} + +concurrent_delete_after + +function concurrent_delete_rollback() +{ + $CLICKHOUSE_CLIENT -q "select 'concurrent_delete_rollback'" + + reset_table tt + + tx 31 "begin transaction" + tx 31 "select count() from tt" + tx 32 "begin transaction" + tx 32 "alter table tt delete where n%2=1" + tx 31 "select count() from tt" + tx 32 "select count() from tt" + tx 31 "select count() from tt" + tx 32 "rollback" + tx 31 "truncate table tt" + tx 31 "commit" + + $CLICKHOUSE_CLIENT -q "select count() from tt" +} + +concurrent_delete_rollback + + +function concurrent_optimize_table_not_start() +{ + $CLICKHOUSE_CLIENT -q "select 'concurrent_optimize_table_not_start'" + + reset_table tt + + tx 41 "begin transaction" + tx 41 "insert into tt values (4)" # inserts all_2_2_0 + + tx 42 "begin transaction" + tx 42 "optimize table tt final" + tx 42 "commit" + + tx 41 "select count() from tt" + tx 41 "commit" + + $CLICKHOUSE_CLIENT -q "select count(), _part from tt group by _part order by _part" +} + +concurrent_optimize_table_not_start + + +function concurrent_optimize_table() +{ + $CLICKHOUSE_CLIENT -q "select 'concurrent_optimize_table'" + + reset_table tt + + $CLICKHOUSE_CLIENT -q "insert into $table values (4), (5)" # inserts all_2_2_0 + + tx 41 "begin transaction" + tx 41 "optimize table tt final" + + tx 42 "begin transaction" + tx 42 "insert into tt values (6)" # inserts all_3_3_0 + + tx 43 "begin transaction" + tx 43 "select count() from tt" + tx 43 "alter table tt drop partition id 'all'" | grep -Eo "SERIALIZATION_ERROR" | uniq + + tx 42 "commit" + tx 43 "commit" | grep -Eo "INVALID_TRANSACTION" | uniq + tx 41 "commit" + + $CLICKHOUSE_CLIENT -q "select count(), _part from tt group by _part order by _part" +} + +concurrent_optimize_table + +function concurrent_optimize_table_before() +{ + $CLICKHOUSE_CLIENT -q "select 'concurrent_optimize_table_before'" + + reset_table tt + + tx 51 "begin transaction" + tx 52 "begin transaction" + tx 51 "optimize table tt final" # inserts all_1_1_1 + tx 51 "rollback" # inserts all_1_1_1 is outdated + tx 52 "alter table tt drop partition id 'all'" | grep -vwe "PART_IS_TEMPORARILY_LOCKED" ||: # conflict with all_1_1_1 + tx 52 "rollback" + + $CLICKHOUSE_CLIENT -q "select count(), _part from tt group by _part order by _part" +} + +concurrent_optimize_table_before + +function drop_parts_which_already_outdated() +{ + $CLICKHOUSE_CLIENT -q "select 'drop_parts_which_already_outdated'" + + reset_table tt "settings old_parts_lifetime=0" + + $CLICKHOUSE_CLIENT -q "optimize table tt final /*all_1_1_1*/" + $CLICKHOUSE_CLIENT -q "optimize table tt final /*all_1_1_2*/" + $CLICKHOUSE_CLIENT -q "optimize table tt final /*all_1_1_3*/" + $CLICKHOUSE_CLIENT -q "optimize table tt final /*all_1_1_4*/" + $CLICKHOUSE_CLIENT -q "optimize table tt final /*all_1_1_5*/" + $CLICKHOUSE_CLIENT -q "optimize table tt final /*all_1_1_6*/" + + $CLICKHOUSE_CLIENT -q "insert into $table values (4)" # inserts all_2_2_0 + + tx 69 "begin transaction" + tx 69 "select 'before optimize', count(), _part from tt group by _part order by _part" + + tx 61 "begin transaction" + tx 61 "optimize table tt final /*all_1_2_7*/" + tx 61 "commit" + + tx 62 "begin transaction" + tx 62 "optimize table tt final /*all_1_2_8*/" + + tx 69 "select 'after optimize', count(), _part from tt group by _part order by _part" + tx 69 "alter table tt drop partition id 'all'" | grep -Eo "SERIALIZATION_ERROR" | uniq + tx 69 "rollback" + + tx 62 "rollback" + + $CLICKHOUSE_CLIENT -q "select 'at the end', count(), _part from tt group by _part order by _part" +} + +drop_parts_which_already_outdated + +function unable_drop_one_part_which_outdated_but_visible() +{ + $CLICKHOUSE_CLIENT -q "select 'unable_drop_one_part_which_outdated_but_visible'" + + reset_table tt "settings old_parts_lifetime=0" + + $CLICKHOUSE_CLIENT -q "optimize table tt final /*all_1_1_1*/" + $CLICKHOUSE_CLIENT -q "optimize table tt final /*all_1_1_2*/" + + $CLICKHOUSE_CLIENT -q "insert into $table values (4)" # inserts all_2_2_0 + + tx 79 "begin transaction" + tx 79 "select 'before optimize', count(), _part from tt group by _part order by _part" + + tx 71 "begin transaction" + tx 71 "optimize table tt final /*all_1_2_3*/" + + tx 79 "select 'after optimize', count(), _part from tt group by _part order by _part" + tx 79 "alter table tt drop part 'all_2_2_0'" | grep -Eo "NO_SUCH_DATA_PART" | uniq + tx 79 "rollback" + + tx 71 "rollback" + + $CLICKHOUSE_CLIENT -q "select 'at the end', count(), _part from tt group by _part order by _part" +} + +unable_drop_one_part_which_outdated_but_visible + +function drop_one_part_which_outdated_and_reverted() +{ + $CLICKHOUSE_CLIENT -q "select 'drop_one_part_which_outdated_and_reverted'" + + reset_table tt "settings old_parts_lifetime=0" + + $CLICKHOUSE_CLIENT -q "optimize table tt final /*all_1_1_1*/" + + $CLICKHOUSE_CLIENT -q "insert into $table values (4)" # inserts all_2_2_0 + + tx 89 "begin transaction" + tx 89 "select 'before optimize', count(), _part from tt group by _part order by _part" + + tx 81 "begin transaction" + tx 81 "optimize table tt final /*all_1_2_2*/" + + tx 89 "select 'after optimize', count(), _part from tt group by _part order by _part" + tx 81 "rollback" + + tx 89 "select 'after rollback', count(), _part from tt group by _part order by _part" + tx 89 "alter table tt drop part 'all_2_2_0'" + tx 89 "commit" + + $CLICKHOUSE_CLIENT -q "select 'at the end', count(), _part from tt group by _part order by _part" +} + +drop_one_part_which_outdated_and_reverted + +function drop_one_part_which_outdated_and_reverted_no_name_intersection() +{ + $CLICKHOUSE_CLIENT -q "select 'drop_one_part_which_outdated_and_reverted_no_name_intersection'" + + reset_table tt "settings old_parts_lifetime=0" + + $CLICKHOUSE_CLIENT -q "insert into $table values (4)" # inserts all_2_2_0 + + tx 99 "begin transaction" + tx 99 "select 'before optimize', count(), _part from tt group by _part order by _part" + + tx 91 "begin transaction" + tx 91 "optimize table tt final /*all_1_2_1*/" + + tx 99 "select 'after optimize', count(), _part from tt group by _part order by _part" + tx 91 "rollback" + + tx 99 "select 'after rollback', count(), _part from tt group by _part order by _part" + tx 99 "alter table tt drop part 'all_2_2_0'" + tx 99 "commit" + + $CLICKHOUSE_CLIENT -q "select 'at the end', count(), _part from tt group by _part order by _part" +} + +drop_one_part_which_outdated_and_reverted_no_name_intersection diff --git a/tests/queries/0_stateless/02422_insert_different_granularity.reference b/tests/queries/0_stateless/02422_insert_different_granularity.reference new file mode 100644 index 00000000000..f4ca728d701 --- /dev/null +++ b/tests/queries/0_stateless/02422_insert_different_granularity.reference @@ -0,0 +1,4 @@ +=== ataptive granularity: table one -; table two + === +=== ataptive granularity: table one -; table two - === +=== ataptive granularity: table one +; table two + === +=== ataptive granularity: table one +; table two - === diff --git a/tests/queries/0_stateless/02422_insert_different_granularity.sql b/tests/queries/0_stateless/02422_insert_different_granularity.sql new file mode 100644 index 00000000000..e122cd134fe --- /dev/null +++ b/tests/queries/0_stateless/02422_insert_different_granularity.sql @@ -0,0 +1,81 @@ +SELECT '=== ataptive granularity: table one -; table two + ==='; + +DROP TABLE IF EXISTS table_one; +CREATE TABLE table_one (id UInt64, value UInt64) +ENGINE = MergeTree +PARTITION BY id +ORDER BY value +SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 100; + +DROP TABLE IF EXISTS table_two; +CREATE TABLE table_two (id UInt64, value UInt64) +ENGINE = MergeTree +PARTITION BY id +ORDER BY value +SETTINGS index_granularity = 8192, index_granularity_bytes = 1024, min_bytes_for_wide_part = 100; + +INSERT INTO table_one SELECT intDiv(number, 10), number FROM numbers(100); + +ALTER TABLE table_two REPLACE PARTITION 0 FROM table_one; + +SELECT '=== ataptive granularity: table one -; table two - ==='; + +DROP TABLE IF EXISTS table_one; + +CREATE TABLE table_one (id UInt64, value UInt64) +ENGINE = MergeTree +PARTITION BY id +ORDER BY value +SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 100; + +DROP TABLE IF EXISTS table_two; + +CREATE TABLE table_two (id UInt64, value UInt64) +ENGINE = MergeTree +PARTITION BY id +ORDER BY value +SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 100; + +INSERT INTO table_one SELECT intDiv(number, 10), number FROM numbers(100); + +ALTER TABLE table_two REPLACE PARTITION 0 FROM table_one; + +SELECT '=== ataptive granularity: table one +; table two + ==='; + +DROP TABLE IF EXISTS table_one; +CREATE TABLE table_one (id UInt64, value UInt64) +ENGINE = MergeTree +PARTITION BY id +ORDER BY value +SETTINGS index_granularity = 8192, index_granularity_bytes = 1024, min_bytes_for_wide_part = 100; + +DROP TABLE IF EXISTS table_two; +CREATE TABLE table_two (id UInt64, value UInt64) +ENGINE = MergeTree +PARTITION BY id +ORDER BY value +SETTINGS index_granularity = 8192, index_granularity_bytes = 1024, min_bytes_for_wide_part = 100; + +INSERT INTO table_one SELECT intDiv(number, 10), number FROM numbers(100); + +ALTER TABLE table_two REPLACE PARTITION 0 FROM table_one; + +SELECT '=== ataptive granularity: table one +; table two - ==='; + +DROP TABLE IF EXISTS table_one; +CREATE TABLE table_one (id UInt64, value UInt64) +ENGINE = MergeTree +PARTITION BY id +ORDER BY value +SETTINGS index_granularity = 8192, index_granularity_bytes = 1024, min_bytes_for_wide_part = 100; + +DROP TABLE IF EXISTS table_two; +CREATE TABLE table_two (id UInt64, value UInt64) +ENGINE = MergeTree +PARTITION BY id +ORDER BY value +SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 100; + +INSERT INTO table_one SELECT intDiv(number, 10), number FROM numbers(100); + +ALTER TABLE table_two REPLACE PARTITION 0 FROM table_one; -- { serverError 36 } diff --git a/tests/queries/0_stateless/02423_drop_memory_parts.reference b/tests/queries/0_stateless/02423_drop_memory_parts.reference new file mode 100644 index 00000000000..d69a5f07a05 --- /dev/null +++ b/tests/queries/0_stateless/02423_drop_memory_parts.reference @@ -0,0 +1,14 @@ +init state +30 +0_1_1_0 InMemory 10 1 +1_2_2_0 InMemory 10 1 +2_3_3_0 InMemory 10 1 +drop part 0 +20 +1_2_2_0 InMemory 10 1 +2_3_3_0 InMemory 10 1 +detach table +attach table +20 +1_2_2_0 InMemory 10 1 +2_3_3_0 InMemory 10 1 diff --git a/tests/queries/0_stateless/02423_drop_memory_parts.sql b/tests/queries/0_stateless/02423_drop_memory_parts.sql new file mode 100644 index 00000000000..0d42847f6e5 --- /dev/null +++ b/tests/queries/0_stateless/02423_drop_memory_parts.sql @@ -0,0 +1,38 @@ +DROP TABLE IF EXISTS table_in_memory; + +CREATE TABLE table_in_memory +( + `id` UInt64, + `value` UInt64 +) +ENGINE = MergeTree +PARTITION BY id +ORDER BY value +SETTINGS min_bytes_for_wide_part=1000, min_bytes_for_compact_part=900; + +SELECT 'init state'; +INSERT INTO table_in_memory SELECT intDiv(number, 10), number FROM numbers(30); + +SELECT count() FROM table_in_memory; +SELECT name, part_type, rows, active from system.parts +WHERE table='table_in_memory' AND database=currentDatabase(); + +SELECT 'drop part 0'; +ALTER TABLE table_in_memory DROP PARTITION 0; + +SELECT count() FROM table_in_memory; +SELECT name, part_type, rows, active from system.parts +WHERE table='table_in_memory' AND database=currentDatabase() AND active; + +SELECT 'detach table'; +DETACH TABLE table_in_memory; + +SELECT name, part_type, rows, active from system.parts +WHERE table='table_in_memory' AND database=currentDatabase(); + +SELECT 'attach table'; +ATTACH TABLE table_in_memory; + +SELECT count() FROM table_in_memory; +SELECT name, part_type, rows, active from system.parts +WHERE table='table_in_memory' AND database=currentDatabase(); diff --git a/tests/queries/0_stateless/02431_single_value_or_null_empty.reference b/tests/queries/0_stateless/02431_single_value_or_null_empty.reference new file mode 100644 index 00000000000..50d25a40af1 --- /dev/null +++ b/tests/queries/0_stateless/02431_single_value_or_null_empty.reference @@ -0,0 +1,5 @@ +\N + +\N +0 \N \N \N +0 \N \N \N diff --git a/tests/queries/0_stateless/02431_single_value_or_null_empty.sql b/tests/queries/0_stateless/02431_single_value_or_null_empty.sql new file mode 100644 index 00000000000..50d7e1a4a8d --- /dev/null +++ b/tests/queries/0_stateless/02431_single_value_or_null_empty.sql @@ -0,0 +1,33 @@ +select singleValueOrNull(number) from numbers(0) with totals; + +SELECT + 0.5 IN ( + SELECT singleValueOrNull(*) + FROM + ( + SELECT 1048577 + FROM numbers(0) + ) +WITH TOTALS + ), + NULL, + NULL NOT IN ( +SELECT + 2147483647, + 1024 IN ( + SELECT + [NULL, 2147483648, NULL, NULL], + number + FROM numbers(7, 100) + ), + [NULL, NULL, NULL, NULL, NULL], + number +FROM numbers(1048576) +WHERE NULL + ), + NULL NOT IN ( +SELECT number +FROM numbers(0) + ) +GROUP BY NULL +WITH CUBE; diff --git a/tests/queries/0_stateless/02470_mutation_sync_race.reference b/tests/queries/0_stateless/02470_mutation_sync_race.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02470_mutation_sync_race.sh b/tests/queries/0_stateless/02470_mutation_sync_race.sh new file mode 100755 index 00000000000..6c259e46cb1 --- /dev/null +++ b/tests/queries/0_stateless/02470_mutation_sync_race.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Tags: long, zookeeper + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT -q "drop table if exists src;" +$CLICKHOUSE_CLIENT -q "create table src(A UInt64) Engine=ReplicatedMergeTree('/clickhouse/{database}/test/src1', '1') order by tuple() SETTINGS min_bytes_for_wide_part=0;" +$CLICKHOUSE_CLIENT -q "insert into src values (0)" + +function thread() +{ + for i in $(seq 1000); do + $CLICKHOUSE_CLIENT -q "alter table src detach partition tuple()" + $CLICKHOUSE_CLIENT -q "alter table src attach partition tuple()" + $CLICKHOUSE_CLIENT -q "alter table src update A = ${i} where 1 settings mutations_sync=2" + $CLICKHOUSE_CLIENT -q "select throwIf(A != ${i}) from src format Null" + done +} + +export -f thread; + +TIMEOUT=30 + +timeout $TIMEOUT bash -c thread || true diff --git a/tests/queries/0_stateless/02475_precise_decimal_arithmetics.reference b/tests/queries/0_stateless/02475_precise_decimal_arithmetics.reference new file mode 100644 index 00000000000..6ffc8602640 --- /dev/null +++ b/tests/queries/0_stateless/02475_precise_decimal_arithmetics.reference @@ -0,0 +1,23 @@ +0 +0 +0 +9999999999999999550522436926092261716351992671467843175339166479588690755584 +9999999999999999451597035424131548206707486713696660676795842648250000000000 +11.126038 +10.8 +-11.126038 +-10.8 +10.8 +1376.638914 +1403.6 +-1376.638914 +-1403.6 +1403.6 +332833500 +999 +1000 +1000 +1000 +0.1 +0.1 +0.1 diff --git a/tests/queries/0_stateless/02475_precise_decimal_arithmetics.sql b/tests/queries/0_stateless/02475_precise_decimal_arithmetics.sql new file mode 100644 index 00000000000..3bd7906c7d8 --- /dev/null +++ b/tests/queries/0_stateless/02475_precise_decimal_arithmetics.sql @@ -0,0 +1,45 @@ +-- Tags: no-fasttest + +-- check cases when one of operands is zero +SELECT divideDecimal(toDecimal32(0, 2), toDecimal128(11.123456, 6)); +SELECT divideDecimal(toDecimal64(123.123, 3), toDecimal64(0, 1)); -- { serverError 153 } +SELECT multiplyDecimal(toDecimal32(0, 2), toDecimal128(11.123456, 6)); +SELECT multiplyDecimal(toDecimal32(123.123, 3), toDecimal128(0, 1)); + +-- don't look at strange query result -- it happens due to bad float precision: toUInt256(1e38) == 99999999999999997752612184630461283328 +SELECT multiplyDecimal(toDecimal256(1e38, 0), toDecimal256(1e38, 0)); +SELECT divideDecimal(toDecimal256(1e66, 0), toDecimal256(1e-10, 10), 0); + +-- fits Decimal256, but scale is too big to fit +SELECT multiplyDecimal(toDecimal256(1e38, 0), toDecimal256(1e38, 0), 2); -- { serverError 407 } +SELECT divideDecimal(toDecimal256(1e72, 0), toDecimal256(1e-5, 5), 2); -- { serverError 407 } + +-- does not fit Decimal256 +SELECT multiplyDecimal(toDecimal256('1e38', 0), toDecimal256('1e38', 0)); -- { serverError 407 } +SELECT multiplyDecimal(toDecimal256(1e39, 0), toDecimal256(1e39, 0), 0); -- { serverError 407 } +SELECT divideDecimal(toDecimal256(1e39, 0), toDecimal256(1e-38, 39)); -- { serverError 407 } + +-- test different signs +SELECT divideDecimal(toDecimal128(123.76, 2), toDecimal128(11.123456, 6)); +SELECT divideDecimal(toDecimal32(123.123, 3), toDecimal128(11.4, 1), 2); +SELECT divideDecimal(toDecimal128(-123.76, 2), toDecimal128(11.123456, 6)); +SELECT divideDecimal(toDecimal32(123.123, 3), toDecimal128(-11.4, 1), 2); +SELECT divideDecimal(toDecimal32(-123.123, 3), toDecimal128(-11.4, 1), 2); + +SELECT multiplyDecimal(toDecimal64(123.76, 2), toDecimal128(11.123456, 6)); +SELECT multiplyDecimal(toDecimal32(123.123, 3), toDecimal128(11.4, 1), 2); +SELECT multiplyDecimal(toDecimal64(-123.76, 2), toDecimal128(11.123456, 6)); +SELECT multiplyDecimal(toDecimal32(123.123, 3), toDecimal128(-11.4, 1), 2); +SELECT multiplyDecimal(toDecimal32(-123.123, 3), toDecimal128(-11.4, 1), 2); + +-- check against non-const columns +SELECT sum(multiplyDecimal(toDecimal64(number, 1), toDecimal64(number, 5))) FROM numbers(1000); +SELECT sum(divideDecimal(toDecimal64(number, 1), toDecimal64(number, 5))) FROM (select * from numbers(1000) OFFSET 1); + +-- check against Nullable type +SELECT multiplyDecimal(toNullable(toDecimal64(10, 1)), toDecimal64(100, 5)); +SELECT multiplyDecimal(toDecimal64(10, 1), toNullable(toDecimal64(100, 5))); +SELECT multiplyDecimal(toNullable(toDecimal64(10, 1)), toNullable(toDecimal64(100, 5))); +SELECT divideDecimal(toNullable(toDecimal64(10, 1)), toDecimal64(100, 5)); +SELECT divideDecimal(toDecimal64(10, 1), toNullable(toDecimal64(100, 5))); +SELECT divideDecimal(toNullable(toDecimal64(10, 1)), toNullable(toDecimal64(100, 5))); diff --git a/tests/queries/0_stateless/02476_fuse_sum_count.sql b/tests/queries/0_stateless/02476_fuse_sum_count.sql index 8ba096013a6..ee65d32d0cf 100644 --- a/tests/queries/0_stateless/02476_fuse_sum_count.sql +++ b/tests/queries/0_stateless/02476_fuse_sum_count.sql @@ -32,4 +32,7 @@ SELECT sum(x), count(x), avg(x) FROM (SELECT number :: Decimal32(0) AS x FROM nu SELECT sum(x), count(x), avg(x), toTypeName(sum(x)), toTypeName(count(x)), toTypeName(avg(x)) FROM (SELECT number :: Decimal32(0) AS x FROM numbers(10)) SETTINGS optimize_syntax_fuse_functions = 0; SELECT sum(x), count(x), avg(x), toTypeName(sum(x)), toTypeName(count(x)), toTypeName(avg(x)) FROM (SELECT number :: Decimal32(0) AS x FROM numbers(10)); +-- TODO: uncomment after https://github.com/ClickHouse/ClickHouse/pull/43372 +-- SELECT avg(b), x - 2 AS b FROM (SELECT number as x FROM numbers(1)) GROUP BY x; + DROP TABLE fuse_tbl; diff --git a/tests/queries/0_stateless/02477_exists_fuzz_43478.reference b/tests/queries/0_stateless/02477_exists_fuzz_43478.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02477_exists_fuzz_43478.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02477_exists_fuzz_43478.sql b/tests/queries/0_stateless/02477_exists_fuzz_43478.sql new file mode 100644 index 00000000000..8ec876eb252 --- /dev/null +++ b/tests/queries/0_stateless/02477_exists_fuzz_43478.sql @@ -0,0 +1,3 @@ +create table test_rows_compact_part__fuzz_11 (x UInt32) engine = MergeTree order by x; +insert into test_rows_compact_part__fuzz_11 select 1; +select 1 from test_rows_compact_part__fuzz_11 where exists(select 1) settings allow_experimental_analyzer=1; diff --git a/tests/queries/0_stateless/02477_fuse_quantiles.reference b/tests/queries/0_stateless/02477_fuse_quantiles.reference index 0938e9f6f6d..7c7d581f7fb 100644 --- a/tests/queries/0_stateless/02477_fuse_quantiles.reference +++ b/tests/queries/0_stateless/02477_fuse_quantiles.reference @@ -1,89 +1,7 @@ 799.2 Nullable(Float64) 899.1 Nullable(Float64) 800.2 Float64 900.1 Float64 +800.2 Float64 100.9 Float64 498.5 500.5 800.2 801.2 900.1 -QUERY id: 0 - PROJECTION COLUMNS - quantile(minus(a, 1)) Nullable(Float64) - plus(quantile(minus(b, 1)), 1) Float64 - plus(quantile(0.8)(minus(b, 1)), 1) Float64 - plus(quantile(0.8)(minus(b, 1)), 2) Float64 - plus(quantile(0.9)(minus(b, 1)), 1) Float64 - PROJECTION - LIST id: 1, nodes: 5 - FUNCTION id: 2, function_name: quantile, function_type: aggregate, result_type: Nullable(Float64) - ARGUMENTS - LIST id: 3, nodes: 1 - FUNCTION id: 4, function_name: minus, function_type: ordinary, result_type: Nullable(Int64) - ARGUMENTS - LIST id: 5, nodes: 2 - COLUMN id: 6, column_name: a, result_type: Nullable(Int32), source_id: 7 - CONSTANT id: 8, constant_value: UInt64_1, constant_value_type: UInt8 - FUNCTION id: 9, function_name: plus, function_type: ordinary, result_type: Float64 - ARGUMENTS - LIST id: 10, nodes: 2 - FUNCTION id: 11, function_name: arrayElement, function_type: ordinary, result_type: Float64 - ARGUMENTS - LIST id: 12, nodes: 2 - FUNCTION id: 13, function_name: quantiles, function_type: aggregate, result_type: Array(Float64) - ARGUMENTS - LIST id: 14, nodes: 1 - FUNCTION id: 15, function_name: minus, function_type: ordinary, result_type: Int64 - ARGUMENTS - LIST id: 16, nodes: 2 - COLUMN id: 17, column_name: b, result_type: Int32, source_id: 7 - CONSTANT id: 18, constant_value: UInt64_1, constant_value_type: UInt8 - CONSTANT id: 19, constant_value: UInt64_1, constant_value_type: UInt8 - CONSTANT id: 20, constant_value: UInt64_1, constant_value_type: UInt8 - FUNCTION id: 21, function_name: plus, function_type: ordinary, result_type: Float64 - ARGUMENTS - LIST id: 22, nodes: 2 - FUNCTION id: 23, function_name: arrayElement, function_type: ordinary, result_type: Float64 - ARGUMENTS - LIST id: 24, nodes: 2 - FUNCTION id: 13, function_name: quantiles, function_type: aggregate, result_type: Array(Float64) - ARGUMENTS - LIST id: 14, nodes: 1 - FUNCTION id: 15, function_name: minus, function_type: ordinary, result_type: Int64 - ARGUMENTS - LIST id: 16, nodes: 2 - COLUMN id: 17, column_name: b, result_type: Int32, source_id: 7 - CONSTANT id: 18, constant_value: UInt64_1, constant_value_type: UInt8 - CONSTANT id: 25, constant_value: UInt64_2, constant_value_type: UInt8 - CONSTANT id: 26, constant_value: UInt64_1, constant_value_type: UInt8 - FUNCTION id: 27, function_name: plus, function_type: ordinary, result_type: Float64 - ARGUMENTS - LIST id: 28, nodes: 2 - FUNCTION id: 29, function_name: arrayElement, function_type: ordinary, result_type: Float64 - ARGUMENTS - LIST id: 30, nodes: 2 - FUNCTION id: 13, function_name: quantiles, function_type: aggregate, result_type: Array(Float64) - ARGUMENTS - LIST id: 14, nodes: 1 - FUNCTION id: 15, function_name: minus, function_type: ordinary, result_type: Int64 - ARGUMENTS - LIST id: 16, nodes: 2 - COLUMN id: 17, column_name: b, result_type: Int32, source_id: 7 - CONSTANT id: 18, constant_value: UInt64_1, constant_value_type: UInt8 - CONSTANT id: 31, constant_value: UInt64_3, constant_value_type: UInt8 - CONSTANT id: 32, constant_value: UInt64_2, constant_value_type: UInt8 - FUNCTION id: 33, function_name: plus, function_type: ordinary, result_type: Float64 - ARGUMENTS - LIST id: 34, nodes: 2 - FUNCTION id: 35, function_name: arrayElement, function_type: ordinary, result_type: Float64 - ARGUMENTS - LIST id: 36, nodes: 2 - FUNCTION id: 13, function_name: quantiles, function_type: aggregate, result_type: Array(Float64) - ARGUMENTS - LIST id: 14, nodes: 1 - FUNCTION id: 15, function_name: minus, function_type: ordinary, result_type: Int64 - ARGUMENTS - LIST id: 16, nodes: 2 - COLUMN id: 17, column_name: b, result_type: Int32, source_id: 7 - CONSTANT id: 18, constant_value: UInt64_1, constant_value_type: UInt8 - CONSTANT id: 37, constant_value: UInt64_4, constant_value_type: UInt8 - CONSTANT id: 38, constant_value: UInt64_1, constant_value_type: UInt8 - JOIN TREE - TABLE id: 7, table_name: default.fuse_tbl 501.5 501.5 QUERY id: 0 PROJECTION COLUMNS @@ -95,54 +13,70 @@ QUERY id: 0 ARGUMENTS LIST id: 3, nodes: 2 FUNCTION id: 4, function_name: quantiles, function_type: aggregate, result_type: Array(Float64) + PARAMETERS + LIST id: 5, nodes: 2 + CONSTANT id: 6, constant_value: Float64_0.5, constant_value_type: Float64 + CONSTANT id: 7, constant_value: Float64_0.9, constant_value_type: Float64 ARGUMENTS - LIST id: 5, nodes: 1 - COLUMN id: 6, column_name: b, result_type: Float64, source_id: 7 - CONSTANT id: 8, constant_value: UInt64_1, constant_value_type: UInt8 - FUNCTION id: 9, function_name: arrayElement, function_type: ordinary, result_type: Float64 + LIST id: 8, nodes: 1 + COLUMN id: 9, column_name: b, result_type: Float64, source_id: 10 + CONSTANT id: 11, constant_value: UInt64_1, constant_value_type: UInt8 + FUNCTION id: 12, function_name: arrayElement, function_type: ordinary, result_type: Float64 ARGUMENTS - LIST id: 10, nodes: 2 + LIST id: 13, nodes: 2 FUNCTION id: 4, function_name: quantiles, function_type: aggregate, result_type: Array(Float64) + PARAMETERS + LIST id: 5, nodes: 2 + CONSTANT id: 6, constant_value: Float64_0.5, constant_value_type: Float64 + CONSTANT id: 7, constant_value: Float64_0.9, constant_value_type: Float64 ARGUMENTS - LIST id: 5, nodes: 1 - COLUMN id: 6, column_name: b, result_type: Float64, source_id: 7 - CONSTANT id: 11, constant_value: UInt64_2, constant_value_type: UInt8 + LIST id: 8, nodes: 1 + COLUMN id: 9, column_name: b, result_type: Float64, source_id: 10 + CONSTANT id: 14, constant_value: UInt64_2, constant_value_type: UInt8 JOIN TREE - QUERY id: 7, is_subquery: 1 + QUERY id: 10, is_subquery: 1 PROJECTION COLUMNS b Float64 PROJECTION - LIST id: 12, nodes: 1 - FUNCTION id: 13, function_name: plus, function_type: ordinary, result_type: Float64 + LIST id: 15, nodes: 1 + FUNCTION id: 16, function_name: plus, function_type: ordinary, result_type: Float64 ARGUMENTS - LIST id: 14, nodes: 2 - COLUMN id: 15, column_name: x, result_type: Float64, source_id: 16 - CONSTANT id: 17, constant_value: UInt64_1, constant_value_type: UInt8 + LIST id: 17, nodes: 2 + COLUMN id: 18, column_name: x, result_type: Float64, source_id: 19 + CONSTANT id: 20, constant_value: UInt64_1, constant_value_type: UInt8 JOIN TREE - QUERY id: 16, is_subquery: 1 + QUERY id: 19, is_subquery: 1 PROJECTION COLUMNS x Float64 quantile(0.9)(b) Float64 PROJECTION - LIST id: 18, nodes: 2 - FUNCTION id: 19, function_name: arrayElement, function_type: ordinary, result_type: Float64 + LIST id: 21, nodes: 2 + FUNCTION id: 22, function_name: arrayElement, function_type: ordinary, result_type: Float64 ARGUMENTS - LIST id: 20, nodes: 2 - FUNCTION id: 21, function_name: quantiles, function_type: aggregate, result_type: Array(Float64) + LIST id: 23, nodes: 2 + FUNCTION id: 24, function_name: quantiles, function_type: aggregate, result_type: Array(Float64) + PARAMETERS + LIST id: 25, nodes: 2 + CONSTANT id: 26, constant_value: Float64_0.5, constant_value_type: Float64 + CONSTANT id: 27, constant_value: Float64_0.9, constant_value_type: Float64 ARGUMENTS - LIST id: 22, nodes: 1 - COLUMN id: 23, column_name: b, result_type: Int32, source_id: 24 - CONSTANT id: 25, constant_value: UInt64_1, constant_value_type: UInt8 - FUNCTION id: 26, function_name: arrayElement, function_type: ordinary, result_type: Float64 + LIST id: 28, nodes: 1 + COLUMN id: 29, column_name: b, result_type: Int32, source_id: 30 + CONSTANT id: 31, constant_value: UInt64_1, constant_value_type: UInt8 + FUNCTION id: 32, function_name: arrayElement, function_type: ordinary, result_type: Float64 ARGUMENTS - LIST id: 27, nodes: 2 - FUNCTION id: 21, function_name: quantiles, function_type: aggregate, result_type: Array(Float64) + LIST id: 33, nodes: 2 + FUNCTION id: 24, function_name: quantiles, function_type: aggregate, result_type: Array(Float64) + PARAMETERS + LIST id: 25, nodes: 2 + CONSTANT id: 26, constant_value: Float64_0.5, constant_value_type: Float64 + CONSTANT id: 27, constant_value: Float64_0.9, constant_value_type: Float64 ARGUMENTS - LIST id: 22, nodes: 1 - COLUMN id: 23, column_name: b, result_type: Int32, source_id: 24 - CONSTANT id: 28, constant_value: UInt64_2, constant_value_type: UInt8 + LIST id: 28, nodes: 1 + COLUMN id: 29, column_name: b, result_type: Int32, source_id: 30 + CONSTANT id: 34, constant_value: UInt64_2, constant_value_type: UInt8 JOIN TREE - TABLE id: 24, table_name: default.fuse_tbl + TABLE id: 30, table_name: default.fuse_tbl GROUP BY - LIST id: 29, nodes: 1 - COLUMN id: 15, column_name: x, result_type: Float64, source_id: 16 + LIST id: 35, nodes: 1 + COLUMN id: 18, column_name: x, result_type: Float64, source_id: 19 diff --git a/tests/queries/0_stateless/02477_fuse_quantiles.sql b/tests/queries/0_stateless/02477_fuse_quantiles.sql index b08c7da1f04..efd861ad7f3 100644 --- a/tests/queries/0_stateless/02477_fuse_quantiles.sql +++ b/tests/queries/0_stateless/02477_fuse_quantiles.sql @@ -9,9 +9,9 @@ INSERT INTO fuse_tbl SELECT number, number + 1 FROM numbers(1000); SELECT quantile(0.8)(a), toTypeName(quantile(0.8)(a)), quantile(0.9)(a), toTypeName(quantile(0.9)(a)) FROM fuse_tbl; SELECT quantile(0.8)(b), toTypeName(quantile(0.8)(b)), quantile(0.9)(b), toTypeName(quantile(0.9)(b)) FROM fuse_tbl; -SELECT quantile(a - 1), quantile(b - 1) + 1, quantile(0.8)(b - 1) + 1, quantile(0.8)(b - 1) + 2, quantile(0.9)(b - 1) + 1 FROM fuse_tbl; +SELECT quantile(0.8)(b), toTypeName(quantile(0.8)(b)), quantile(0.1)(b), toTypeName(quantile(0.1)(b)) FROM fuse_tbl; -EXPLAIN QUERY TREE run_passes = 1 SELECT quantile(a - 1), quantile(b - 1) + 1, quantile(0.8)(b - 1) + 1, quantile(0.8)(b - 1) + 2, quantile(0.9)(b - 1) + 1 FROM fuse_tbl; +SELECT quantile(a - 1), quantile(b - 1) + 1, quantile(0.8)(b - 1) + 1, quantile(0.8)(b - 1) + 2, quantile(0.9)(b - 1) + 1 FROM fuse_tbl; SELECT quantile(0.5)(b), quantile(0.9)(b) from (SELECT x + 1 as b FROM (SELECT quantile(0.5)(b) as x, quantile(0.9)(b) FROM fuse_tbl) GROUP BY x); EXPLAIN QUERY TREE run_passes = 1 SELECT quantile(0.5)(b), quantile(0.9)(b) from (SELECT x + 1 as b FROM (SELECT quantile(0.5)(b) as x, quantile(0.9)(b) FROM fuse_tbl) GROUP BY x); diff --git a/tests/queries/0_stateless/02477_single_value_data_string_regression.reference b/tests/queries/0_stateless/02477_single_value_data_string_regression.reference index e89b8ff7d99..9285866de08 100644 --- a/tests/queries/0_stateless/02477_single_value_data_string_regression.reference +++ b/tests/queries/0_stateless/02477_single_value_data_string_regression.reference @@ -23,3 +23,8 @@ 1M without 0 1048576 1M with 0 1048575 fuzz2 0123 4 +1 0 +2 \0 1 +3 \0\0\0\0 4 +4 abrac\0dabra\0 12 +abrac\0dabra\0 12 diff --git a/tests/queries/0_stateless/02477_single_value_data_string_regression.sql b/tests/queries/0_stateless/02477_single_value_data_string_regression.sql index c8030733e34..0f11a06f3fc 100644 --- a/tests/queries/0_stateless/02477_single_value_data_string_regression.sql +++ b/tests/queries/0_stateless/02477_single_value_data_string_regression.sql @@ -90,11 +90,12 @@ SELECT '-1', maxMerge(x), length(maxMerge(x)) from (select CAST(unhex('ffffffff' SELECT '-2', maxMerge(x), length(maxMerge(x)) from (select CAST(unhex('fffffffe') || randomString(100500), 'AggregateFunction(max, String)') as x); SELECT '-2^31', maxMerge(x), length(maxMerge(x)) from (select CAST(unhex('00000080') || randomString(100500), 'AggregateFunction(max, String)') as x); -SELECT '2^31-2', maxMerge(x) from (select CAST(unhex('feffff7f') || randomString(100500), 'AggregateFunction(max, String)') as x); -- { serverError TOO_LARGE_STRING_SIZE } SELECT '2^31-1', maxMerge(x) from (select CAST(unhex('ffffff7f') || randomString(100500), 'AggregateFunction(max, String)') as x); -- { serverError TOO_LARGE_STRING_SIZE } -SELECT '2^30', maxMerge(x) from (select CAST(unhex('00000040') || randomString(100500), 'AggregateFunction(max, String)') as x); -- { serverError TOO_LARGE_STRING_SIZE } -SELECT '2^30+1', maxMerge(x) from (select CAST(unhex('01000040') || randomString(100500), 'AggregateFunction(max, String)') as x); -- { serverError TOO_LARGE_STRING_SIZE } +SELECT '2^31-2', maxMerge(x) from (select CAST(unhex('feffff7f') || randomString(100500), 'AggregateFunction(max, String)') as x); -- { serverError CANNOT_READ_ALL_DATA } + +SELECT '2^30', maxMerge(x) from (select CAST(unhex('00000040') || randomString(100500), 'AggregateFunction(max, String)') as x); -- { serverError CANNOT_READ_ALL_DATA } +SELECT '2^30+1', maxMerge(x) from (select CAST(unhex('01000040') || randomString(100500), 'AggregateFunction(max, String)') as x); -- { serverError CANNOT_READ_ALL_DATA } SELECT '2^30-1', maxMerge(x) from (select CAST(unhex('ffffff3f') || randomString(100500), 'AggregateFunction(max, String)') as x); -- { serverError CANNOT_READ_ALL_DATA } -- The following query works, but it's too long and consumes to much memory @@ -107,3 +108,14 @@ SELECT 'fuzz2', finalizeAggregation(CAST(unhex('04000000' || '30313233' || '01' SELECT 'fuzz3', finalizeAggregation(CAST(unhex('04000000' || '30313233' || '00' || 'ffffffffffffffff'), 'AggregateFunction(argMax, String, UInt64)')) as x, length(x); -- { serverError CORRUPTED_DATA } SELECT 'fuzz4', finalizeAggregation(CAST(unhex('04000000' || '30313233' || '00'), 'AggregateFunction(argMax, String, UInt64)')) as x, length(x); -- { serverError CORRUPTED_DATA } SELECT 'fuzz5', finalizeAggregation(CAST(unhex('0100000000000000000FFFFFFFF0'), 'AggregateFunction(argMax, UInt64, String)')); -- { serverError CORRUPTED_DATA } + + +drop table if exists aggr; +create table aggr (n int, s AggregateFunction(max, String)) engine=MergeTree order by n; +insert into aggr select 1, maxState(''); +insert into aggr select 2, maxState('\0'); +insert into aggr select 3, maxState('\0\0\0\0'); +insert into aggr select 4, maxState('abrac\0dabra\0'); +select n, maxMerge(s) as x, length(x) from aggr group by n order by n; +select maxMerge(s) as x, length(x) from aggr; +drop table aggr; diff --git a/tests/queries/0_stateless/02480_client_option_print_num_processed_rows.expect b/tests/queries/0_stateless/02480_client_option_print_num_processed_rows.expect new file mode 100755 index 00000000000..77e219e804e --- /dev/null +++ b/tests/queries/0_stateless/02480_client_option_print_num_processed_rows.expect @@ -0,0 +1,42 @@ +#!/usr/bin/expect -f + +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + +log_user 0 +set timeout 60 +match_max 100000 +set stty_init "rows 25 cols 120" + +expect_after { + eof { exp_continue } + timeout { exit 1 } +} + +spawn bash +send "source $basedir/../shell_config.sh\r" + +send -- "\$CLICKHOUSE_CLIENT --query 'DROP TABLE IF EXISTS num_processed_rows_test_0' >/dev/null 2>&1\r" + +send -- "\$CLICKHOUSE_CLIENT --query 'CREATE TABLE num_processed_rows_test_0 (val String) ENGINE = Memory;' >/dev/null 2>&1\r" + +### When requested we should get the count on exit: +send -- "\$CLICKHOUSE_CLIENT --processed-rows --query \"INSERT INTO num_processed_rows_test_0 VALUES (\'x\');\" \r" +expect "Processed rows: 1" + +send "yes | head -n7757 | \$CLICKHOUSE_CLIENT --processed-rows --query 'INSERT INTO num_processed_rows_test_0 format TSV\'\r" +expect "Processed rows: 7757" + + + +### By default it should not show up: + +send -- "\$CLICKHOUSE_CLIENT --query \"INSERT INTO num_processed_rows_test_0 VALUES (\'x\');\" && echo OK\r" +expect -exact "OK\r" + +send "yes | head -n7757 | \$CLICKHOUSE_CLIENT --query 'INSERT INTO num_processed_rows_test_0 format TSV\' && echo OK\r" +expect -exact "OK\r" + +send "exit\r" +expect eof diff --git a/tests/queries/0_stateless/02480_client_option_print_num_processed_rows.reference b/tests/queries/0_stateless/02480_client_option_print_num_processed_rows.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02480_suspicious_lowcard_in_key.reference b/tests/queries/0_stateless/02480_suspicious_lowcard_in_key.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02480_suspicious_lowcard_in_key.sql b/tests/queries/0_stateless/02480_suspicious_lowcard_in_key.sql new file mode 100644 index 00000000000..8d537514dbf --- /dev/null +++ b/tests/queries/0_stateless/02480_suspicious_lowcard_in_key.sql @@ -0,0 +1,11 @@ +set allow_suspicious_low_cardinality_types=1; + +drop table if exists test; + +create table test (val LowCardinality(Float32)) engine MergeTree order by val; + +insert into test values (nan); + +select count() from test where toUInt64(val) = -1; -- { serverError 70 } + +drop table if exists test; diff --git a/tests/queries/0_stateless/02481_async_insert_race_long.reference b/tests/queries/0_stateless/02481_async_insert_race_long.reference new file mode 100644 index 00000000000..d86bac9de59 --- /dev/null +++ b/tests/queries/0_stateless/02481_async_insert_race_long.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/02481_async_insert_race_long.sh b/tests/queries/0_stateless/02481_async_insert_race_long.sh new file mode 100755 index 00000000000..cec9278c127 --- /dev/null +++ b/tests/queries/0_stateless/02481_async_insert_race_long.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-fasttest, long + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +export MY_CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --async_insert_busy_timeout_ms 10 --async_insert_max_data_size 1 --async_insert 1" + +function insert1() +{ + while true; do + ${MY_CLICKHOUSE_CLIENT} --wait_for_async_insert 0 -q 'INSERT INTO async_inserts_race FORMAT CSV 1,"a"' + done +} + +function insert2() +{ + while true; do + ${MY_CLICKHOUSE_CLIENT} --wait_for_async_insert 0 -q 'INSERT INTO async_inserts_race FORMAT JSONEachRow {"id": 5, "s": "e"} {"id": 6, "s": "f"}' + done +} + +function insert3() +{ + while true; do + ${MY_CLICKHOUSE_CLIENT} --wait_for_async_insert 1 -q "INSERT INTO async_inserts_race VALUES (7, 'g') (8, 'h')" & + sleep 0.05 + done +} + +function select1() +{ + while true; do + ${MY_CLICKHOUSE_CLIENT} -q "SELECT * FROM async_inserts_race FORMAT Null" + done + +} + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS async_inserts_race" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE async_inserts_race (id UInt32, s String) ENGINE = MergeTree ORDER BY id" + +TIMEOUT=10 + +export -f insert1 +export -f insert2 +export -f insert3 +export -f select1 + +for _ in {1..3}; do + timeout $TIMEOUT bash -c insert1 & + timeout $TIMEOUT bash -c insert2 & + timeout $TIMEOUT bash -c insert3 & +done + +timeout $TIMEOUT bash -c select1 & + +wait +echo "OK" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS async_inserts_race"; diff --git a/tests/queries/0_stateless/02481_i43247_ubsan_in_minmaxany.sql b/tests/queries/0_stateless/02481_i43247_ubsan_in_minmaxany.sql index 7dc29c2daae..c893e49fed3 100644 --- a/tests/queries/0_stateless/02481_i43247_ubsan_in_minmaxany.sql +++ b/tests/queries/0_stateless/02481_i43247_ubsan_in_minmaxany.sql @@ -1,6 +1,6 @@ -- https://github.com/ClickHouse/ClickHouse/issues/43247 SELECT finalizeAggregation(CAST('AggregateFunction(categoricalInformationValue, Nullable(UInt8), UInt8)AggregateFunction(categoricalInformationValue, Nullable(UInt8), UInt8)', - 'AggregateFunction(min, String)')); -- { serverError 131 } + 'AggregateFunction(min, String)')); -- { serverError CANNOT_READ_ALL_DATA } -- Value from hex(minState('0123456789012345678901234567890123456789012345678901234567890123')). Size 63 + 1 (64) SELECT finalizeAggregation(CAST(unhex('4000000030313233343536373839303132333435363738393031323334353637383930313233343536373839303132333435363738393031323334353637383930313233'), diff --git a/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins.reference b/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins.reference new file mode 100644 index 00000000000..ba26d5d21d7 --- /dev/null +++ b/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins.reference @@ -0,0 +1,28 @@ +if with one LC argument +b +a +b +b +a +b +a +if with LC and NULL arguments +\N +a +\N +\N +a +\N +a +if with two LC arguments +b +a +b +b +a +a +a +\N +1 +1 +1 diff --git a/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins.sql b/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins.sql new file mode 100644 index 00000000000..6f33db6aa1e --- /dev/null +++ b/tests/queries/0_stateless/02481_low_cardinality_with_short_circuit_functins.sql @@ -0,0 +1,26 @@ +set short_circuit_function_evaluation='force_enable'; + +select 'if with one LC argument'; +select if(0, toLowCardinality('a'), 'b'); +select if(1, toLowCardinality('a'), 'b'); +select if(materialize(0), materialize(toLowCardinality('a')), materialize('b')); +select if(number % 2, toLowCardinality('a'), 'b') from numbers(2); +select if(number % 2, materialize(toLowCardinality('a')), materialize('b')) from numbers(2); + +select 'if with LC and NULL arguments'; +select if(0, toLowCardinality('a'), NULL); +select if(1, toLowCardinality('a'), NULL); +select if(materialize(0), materialize(toLowCardinality('a')), NULL); +select if(number % 2, toLowCardinality('a'), NULL) from numbers(2); +select if(number % 2, materialize(toLowCardinality('a')), NULL) from numbers(2); + +select 'if with two LC arguments'; +select if(0, toLowCardinality('a'), toLowCardinality('b')); +select if(1, toLowCardinality('a'), toLowCardinality('b')); +select if(materialize(0), materialize(toLowCardinality('a')), materialize(toLowCardinality('b'))); +select if(number % 2, toLowCardinality('a'), toLowCardinality('b')) from numbers(2); +select if(number % 2, materialize(toLowCardinality('a')), materialize(toLowCardinality('a'))) from numbers(2); + +select if(number % 2, toLowCardinality(number), NULL) from numbers(2); +select if(number % 2, toLowCardinality(number), toLowCardinality(number + 1)) from numbers(2); + diff --git a/tests/queries/0_stateless/02481_parquet_int_list_multiple_chunks.reference b/tests/queries/0_stateless/02481_parquet_int_list_multiple_chunks.reference new file mode 100644 index 00000000000..285856e363a --- /dev/null +++ b/tests/queries/0_stateless/02481_parquet_int_list_multiple_chunks.reference @@ -0,0 +1,3 @@ +Parquet +3d94071a2fe62a3b3285f170ca6f42e5 - +70000 diff --git a/tests/queries/0_stateless/02481_parquet_int_list_multiple_chunks.sh b/tests/queries/0_stateless/02481_parquet_int_list_multiple_chunks.sh new file mode 100755 index 00000000000..c2c6f689851 --- /dev/null +++ b/tests/queries/0_stateless/02481_parquet_int_list_multiple_chunks.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +echo "Parquet" + +# File generated with the below script + +#import pyarrow as pa +#import pyarrow.parquet as pq +#import random +# +# +#def gen_array(offset): +# array = [] +# array_length = random.randint(0, 9) +# for i in range(array_length): +# array.append(i + offset) +# +# return array +# +# +#def gen_arrays(number_of_arrays): +# list_of_arrays = [] +# for i in range(number_of_arrays): +# list_of_arrays.append(gen_array(i)) +# return list_of_arrays +# +#arr = pa.array(gen_arrays(70000)) +#table = pa.table([arr], ["arr"]) +#pq.write_table(table, "int-list-zero-based-chunked-array.parquet") + +DATA_FILE=$CUR_DIR/data_parquet/int-list-zero-based-chunked-array.parquet +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_load" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_load (arr Array(Int64)) ENGINE = Memory" +cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "INSERT INTO parquet_load FORMAT Parquet" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_load" | md5sum +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM parquet_load" +${CLICKHOUSE_CLIENT} --query="drop table parquet_load" \ No newline at end of file diff --git a/tests/queries/0_stateless/02481_parquet_list_monotonically_increasing_offsets.reference b/tests/queries/0_stateless/02481_parquet_list_monotonically_increasing_offsets.reference new file mode 100644 index 00000000000..2db066c0f87 --- /dev/null +++ b/tests/queries/0_stateless/02481_parquet_list_monotonically_increasing_offsets.reference @@ -0,0 +1,3 @@ +Parquet +e1cfe4265689ead763b18489b363344d - +39352 diff --git a/tests/queries/0_stateless/02481_parquet_list_monotonically_increasing_offsets.sh b/tests/queries/0_stateless/02481_parquet_list_monotonically_increasing_offsets.sh new file mode 100755 index 00000000000..47245eeb940 --- /dev/null +++ b/tests/queries/0_stateless/02481_parquet_list_monotonically_increasing_offsets.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +echo "Parquet" + +DATA_FILE=$CUR_DIR/data_parquet/list_monotonically_increasing_offsets.parquet +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_load" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_load (list Array(Int64), json Nullable(String)) ENGINE = Memory" +cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "INSERT INTO parquet_load FORMAT Parquet" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_load" | md5sum +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM parquet_load" +${CLICKHOUSE_CLIENT} --query="drop table parquet_load" \ No newline at end of file diff --git a/tests/queries/0_stateless/02481_pk_analysis_with_enum_to_string.reference b/tests/queries/0_stateless/02481_pk_analysis_with_enum_to_string.reference new file mode 100644 index 00000000000..b6a7d89c68e --- /dev/null +++ b/tests/queries/0_stateless/02481_pk_analysis_with_enum_to_string.reference @@ -0,0 +1 @@ +16 diff --git a/tests/queries/0_stateless/02481_pk_analysis_with_enum_to_string.sql b/tests/queries/0_stateless/02481_pk_analysis_with_enum_to_string.sql new file mode 100644 index 00000000000..91402bbed60 --- /dev/null +++ b/tests/queries/0_stateless/02481_pk_analysis_with_enum_to_string.sql @@ -0,0 +1,23 @@ +CREATE TABLE gen +( + repo_name String, + event_type Enum8('CommitCommentEvent' = 1, 'CreateEvent' = 2, 'DeleteEvent' = 3, 'ForkEvent' = 4, 'GollumEvent' = 5, 'IssueCommentEvent' = 6, 'IssuesEvent' = 7, 'MemberEvent' = 8, 'PublicEvent' = 9, 'PullRequestEvent' = 10, 'PullRequestReviewCommentEvent' = 11, 'PushEvent' = 12, 'ReleaseEvent' = 13, 'SponsorshipEvent' = 14, 'WatchEvent' = 15, 'GistEvent' = 16, 'FollowEvent' = 17, 'DownloadEvent' = 18, 'PullRequestReviewEvent' = 19, 'ForkApplyEvent' = 20, 'Event' = 21, 'TeamAddEvent' = 22), + actor_login String, + created_at DateTime, + action Enum8('none' = 0, 'created' = 1, 'added' = 2, 'edited' = 3, 'deleted' = 4, 'opened' = 5, 'closed' = 6, 'reopened' = 7, 'assigned' = 8, 'unassigned' = 9, 'labeled' = 10, 'unlabeled' = 11, 'review_requested' = 12, 'review_request_removed' = 13, 'synchronize' = 14, 'started' = 15, 'published' = 16, 'update' = 17, 'create' = 18, 'fork' = 19, 'merged' = 20), + number UInt32, + merged_at DateTime +) +ENGINE = GenerateRandom; + +CREATE TABLE github_events AS gen ENGINE=MergeTree ORDER BY (event_type, repo_name, created_at); + +INSERT INTO github_events SELECT * FROM gen LIMIT 100000; + +INSERT INTO github_events VALUES ('apache/pulsar','PullRequestEvent','hangc0276','2021-01-22 06:58:03','opened',9276,'1970-01-01 00:00:00') ('apache/pulsar','PullRequestEvent','hangc0276','2021-01-25 02:38:07','closed',9276,'1970-01-01 00:00:00') ('apache/pulsar','PullRequestEvent','hangc0276','2021-01-25 02:38:09','reopened',9276,'1970-01-01 00:00:00') ('apache/pulsar','PullRequestEvent','hangc0276','2021-04-22 06:05:09','closed',9276,'2021-04-22 06:05:08') ('apache/pulsar','IssueCommentEvent','hangc0276','2021-01-23 00:32:09','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','IssueCommentEvent','hangc0276','2021-01-23 02:52:11','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','IssueCommentEvent','hangc0276','2021-01-24 03:02:31','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','IssueCommentEvent','hangc0276','2021-01-25 02:16:42','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','IssueCommentEvent','hangc0276','2021-01-26 06:52:42','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','IssueCommentEvent','hangc0276','2021-01-27 01:10:33','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','IssueCommentEvent','hangc0276','2021-01-29 02:11:41','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','IssueCommentEvent','hangc0276','2021-02-02 07:35:40','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','IssueCommentEvent','hangc0276','2021-02-03 00:44:26','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','IssueCommentEvent','hangc0276','2021-02-03 02:14:26','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','PullRequestReviewEvent','codelipenghui','2021-03-29 14:31:25','created',9276,'1970-01-01 00:00:00') ('apache/pulsar','PullRequestReviewEvent','eolivelli','2021-03-29 16:34:02','created',9276,'1970-01-01 00:00:00'); + +OPTIMIZE TABLE github_events FINAL; + +SELECT count() +FROM github_events +WHERE (repo_name = 'apache/pulsar') AND (toString(event_type) IN ('PullRequestEvent', 'PullRequestReviewCommentEvent', 'PullRequestReviewEvent', 'IssueCommentEvent')) AND (actor_login NOT IN ('github-actions[bot]', 'codecov-commenter')) AND (number = 9276); diff --git a/tests/queries/0_stateless/02481_xxh3_hash_function.reference b/tests/queries/0_stateless/02481_xxh3_hash_function.reference new file mode 100644 index 00000000000..73276fe135e --- /dev/null +++ b/tests/queries/0_stateless/02481_xxh3_hash_function.reference @@ -0,0 +1 @@ +18009318874338624809 diff --git a/tests/queries/0_stateless/02481_xxh3_hash_function.sql b/tests/queries/0_stateless/02481_xxh3_hash_function.sql new file mode 100644 index 00000000000..cd87f08a68e --- /dev/null +++ b/tests/queries/0_stateless/02481_xxh3_hash_function.sql @@ -0,0 +1 @@ +SELECT xxh3('ClickHouse'); diff --git a/tests/queries/0_stateless/02482_json_nested_arrays_with_same_keys.reference b/tests/queries/0_stateless/02482_json_nested_arrays_with_same_keys.reference new file mode 100644 index 00000000000..0bde2d265cf --- /dev/null +++ b/tests/queries/0_stateless/02482_json_nested_arrays_with_same_keys.reference @@ -0,0 +1 @@ +{"list.nested.x.r":[[1,2]],"list.x.r":[[1]]} diff --git a/tests/queries/0_stateless/02482_json_nested_arrays_with_same_keys.sh b/tests/queries/0_stateless/02482_json_nested_arrays_with_same_keys.sh new file mode 100755 index 00000000000..0d0caa78ea3 --- /dev/null +++ b/tests/queries/0_stateless/02482_json_nested_arrays_with_same_keys.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +echo ' +{ + "obj" : + { + "list" : + [ + { + "nested" : { + "x" : [{"r" : 1}, {"r" : 2}] + }, + "x" : [{"r" : 1}] + } + ] + } +}' > 02482_object_data.jsonl + +$CLICKHOUSE_LOCAL --allow_experimental_object_type=1 -q "select * from file(02482_object_data.jsonl, auto, 'obj JSON')" + +rm 02482_object_data.jsonl + diff --git a/tests/queries/0_stateless/02483_substitute_udf_create.reference b/tests/queries/0_stateless/02483_substitute_udf_create.reference new file mode 100644 index 00000000000..ea07b63e068 --- /dev/null +++ b/tests/queries/0_stateless/02483_substitute_udf_create.reference @@ -0,0 +1,33 @@ +-- { echo } +CREATE FUNCTION 02483_plusone AS (a) -> a + 1; +CREATE TABLE 02483_substitute_udf (id UInt32, number UInt32 DEFAULT 02483_plusone(id)) ENGINE=MergeTree() ORDER BY id; +DESC TABLE 02483_substitute_udf; +id UInt32 +number UInt32 DEFAULT id + 1 +INSERT INTO 02483_substitute_udf (id, number) VALUES (1, NULL); +SELECT * FROM 02483_substitute_udf ORDER BY id; +1 2 +CREATE FUNCTION 02483_plustwo AS (a) -> a + 2; +ALTER TABLE 02483_substitute_udf MODIFY COLUMN number UInt32 DEFAULT 02483_plustwo(id); +DESC TABLE 02483_substitute_udf; +id UInt32 +number UInt32 DEFAULT id + 2 +INSERT INTO 02483_substitute_udf (id, number) VALUES (5, NULL); +SELECT * FROM 02483_substitute_udf ORDER BY id; +1 2 +5 7 +CREATE FUNCTION 02483_plusthree AS (a) -> a + 3; +ALTER TABLE 02483_substitute_udf DROP COLUMN number; +ALTER TABLE 02483_substitute_udf ADD COLUMN new_number UInt32 DEFAULT 02483_plusthree(id); +DESC TABLE 02483_substitute_udf; +id UInt32 +new_number UInt32 DEFAULT id + 3 +INSERT INTO 02483_substitute_udf (id, new_number) VALUES (10, NULL); +SELECT * FROM 02483_substitute_udf ORDER BY id; +1 4 +5 8 +10 13 +DROP TABLE 02483_substitute_udf; +DROP FUNCTION 02483_plusone; +DROP FUNCTION 02483_plustwo; +DROP FUNCTION 02483_plusthree; diff --git a/tests/queries/0_stateless/02483_substitute_udf_create.sql b/tests/queries/0_stateless/02483_substitute_udf_create.sql new file mode 100644 index 00000000000..9cfb198cf4c --- /dev/null +++ b/tests/queries/0_stateless/02483_substitute_udf_create.sql @@ -0,0 +1,31 @@ +-- Tags: no-parallel + +DROP TABLE IF EXISTS 02483_substitute_udf; +DROP FUNCTION IF EXISTS 02483_plusone; +DROP FUNCTION IF EXISTS 02483_plustwo; +DROP FUNCTION IF EXISTS 02483_plusthree; + +-- { echo } +CREATE FUNCTION 02483_plusone AS (a) -> a + 1; +CREATE TABLE 02483_substitute_udf (id UInt32, number UInt32 DEFAULT 02483_plusone(id)) ENGINE=MergeTree() ORDER BY id; +DESC TABLE 02483_substitute_udf; +INSERT INTO 02483_substitute_udf (id, number) VALUES (1, NULL); +SELECT * FROM 02483_substitute_udf ORDER BY id; + +CREATE FUNCTION 02483_plustwo AS (a) -> a + 2; +ALTER TABLE 02483_substitute_udf MODIFY COLUMN number UInt32 DEFAULT 02483_plustwo(id); +DESC TABLE 02483_substitute_udf; +INSERT INTO 02483_substitute_udf (id, number) VALUES (5, NULL); +SELECT * FROM 02483_substitute_udf ORDER BY id; + +CREATE FUNCTION 02483_plusthree AS (a) -> a + 3; +ALTER TABLE 02483_substitute_udf DROP COLUMN number; +ALTER TABLE 02483_substitute_udf ADD COLUMN new_number UInt32 DEFAULT 02483_plusthree(id); +DESC TABLE 02483_substitute_udf; +INSERT INTO 02483_substitute_udf (id, new_number) VALUES (10, NULL); +SELECT * FROM 02483_substitute_udf ORDER BY id; + +DROP TABLE 02483_substitute_udf; +DROP FUNCTION 02483_plusone; +DROP FUNCTION 02483_plustwo; +DROP FUNCTION 02483_plusthree; diff --git a/tests/queries/0_stateless/02484_substitute_udf_storage_args.reference b/tests/queries/0_stateless/02484_substitute_udf_storage_args.reference new file mode 100644 index 00000000000..6a799b1e013 --- /dev/null +++ b/tests/queries/0_stateless/02484_substitute_udf_storage_args.reference @@ -0,0 +1,23 @@ +-- { echo } +CREATE TABLE 02484_substitute_udf (id UInt32, dt DateTime, number UInt32) +ENGINE=MergeTree() +ORDER BY 02484_plusone(id) +PARTITION BY 02484_plustwo(id) +SAMPLE BY 02484_plusone(id) +TTL 02484_plusthreemonths(dt); +SHOW CREATE TABLE 02484_substitute_udf; +CREATE TABLE default.`02484_substitute_udf`\n(\n `id` UInt32,\n `dt` DateTime,\n `number` UInt32\n)\nENGINE = MergeTree\nPARTITION BY id + 2\nORDER BY id + 1\nSAMPLE BY id + 1\nTTL dt + toIntervalMonth(3)\nSETTINGS index_granularity = 8192 +CREATE FUNCTION 02484_plusthree AS (a) -> a + 3; +ALTER TABLE 02484_substitute_udf ADD COLUMN id2 UInt64, MODIFY ORDER BY (02484_plusone(id), 02484_plusthree(id2)); +SHOW CREATE TABLE 02484_substitute_udf; +CREATE TABLE default.`02484_substitute_udf`\n(\n `id` UInt32,\n `dt` DateTime,\n `number` UInt32,\n `id2` UInt64\n)\nENGINE = MergeTree\nPARTITION BY id + 2\nPRIMARY KEY id + 1\nORDER BY (id + 1, id2 + 3)\nSAMPLE BY id + 1\nTTL dt + toIntervalMonth(3)\nSETTINGS index_granularity = 8192 +CREATE FUNCTION 02484_plusthreedays AS (a) -> a + INTERVAL 3 DAY; +ALTER TABLE 02484_substitute_udf MODIFY TTL 02484_plusthreedays(dt); +SHOW CREATE TABLE 02484_substitute_udf; +CREATE TABLE default.`02484_substitute_udf`\n(\n `id` UInt32,\n `dt` DateTime,\n `number` UInt32,\n `id2` UInt64\n)\nENGINE = MergeTree\nPARTITION BY id + 2\nPRIMARY KEY id + 1\nORDER BY (id + 1, id2 + 3)\nSAMPLE BY id + 1\nTTL dt + toIntervalDay(3)\nSETTINGS index_granularity = 8192 +DROP TABLE 02484_substitute_udf; +DROP FUNCTION 02484_plusone; +DROP FUNCTION 02484_plustwo; +DROP FUNCTION 02484_plusthree; +DROP FUNCTION 02484_plusthreemonths; +DROP FUNCTION 02484_plusthreedays; diff --git a/tests/queries/0_stateless/02484_substitute_udf_storage_args.sql b/tests/queries/0_stateless/02484_substitute_udf_storage_args.sql new file mode 100644 index 00000000000..a39c6009d58 --- /dev/null +++ b/tests/queries/0_stateless/02484_substitute_udf_storage_args.sql @@ -0,0 +1,37 @@ +-- Tags: no-parallel + +DROP TABLE IF EXISTS 02484_substitute_udf; +DROP FUNCTION IF EXISTS 02484_plusone; +DROP FUNCTION IF EXISTS 02484_plustwo; +DROP FUNCTION IF EXISTS 02484_plusthree; +DROP FUNCTION IF EXISTS 02484_plusthreemonths; +DROP FUNCTION IF EXISTS 02484_plusthreedays; + +CREATE FUNCTION 02484_plusone AS (a) -> a + 1; +CREATE FUNCTION 02484_plustwo AS (a) -> a + 2; +CREATE FUNCTION 02484_plusthreemonths AS (a) -> a + INTERVAL 3 MONTH; + +-- { echo } +CREATE TABLE 02484_substitute_udf (id UInt32, dt DateTime, number UInt32) +ENGINE=MergeTree() +ORDER BY 02484_plusone(id) +PARTITION BY 02484_plustwo(id) +SAMPLE BY 02484_plusone(id) +TTL 02484_plusthreemonths(dt); + +SHOW CREATE TABLE 02484_substitute_udf; + +CREATE FUNCTION 02484_plusthree AS (a) -> a + 3; +ALTER TABLE 02484_substitute_udf ADD COLUMN id2 UInt64, MODIFY ORDER BY (02484_plusone(id), 02484_plusthree(id2)); +SHOW CREATE TABLE 02484_substitute_udf; + +CREATE FUNCTION 02484_plusthreedays AS (a) -> a + INTERVAL 3 DAY; +ALTER TABLE 02484_substitute_udf MODIFY TTL 02484_plusthreedays(dt); +SHOW CREATE TABLE 02484_substitute_udf; + +DROP TABLE 02484_substitute_udf; +DROP FUNCTION 02484_plusone; +DROP FUNCTION 02484_plustwo; +DROP FUNCTION 02484_plusthree; +DROP FUNCTION 02484_plusthreemonths; +DROP FUNCTION 02484_plusthreedays; diff --git a/tests/queries/0_stateless/02491_part_log_has_table_uuid.reference b/tests/queries/0_stateless/02491_part_log_has_table_uuid.reference index 310e84c6fae..fbc09700fe6 100644 --- a/tests/queries/0_stateless/02491_part_log_has_table_uuid.reference +++ b/tests/queries/0_stateless/02491_part_log_has_table_uuid.reference @@ -1,3 +1,4 @@ -NewPart NotAMerge -MergeParts RegularMerge -RemovePart NotAMerge +1 NewPart NotAMerge all_1_1_0 +1 MergeParts RegularMerge all_1_1_1 +1 NewPart NotAMerge all_1_1_2 +1 RemovePart NotAMerge all_1_1_1 diff --git a/tests/queries/0_stateless/02491_part_log_has_table_uuid.sql b/tests/queries/0_stateless/02491_part_log_has_table_uuid.sql index 6291dbab258..1d18962443c 100644 --- a/tests/queries/0_stateless/02491_part_log_has_table_uuid.sql +++ b/tests/queries/0_stateless/02491_part_log_has_table_uuid.sql @@ -1,3 +1,5 @@ +-- Tags: no-ordinary-database + create table data_02491 (key Int) engine=MergeTree() order by tuple(); insert into data_02491 values (1); optimize table data_02491 final; @@ -5,12 +7,16 @@ truncate table data_02491; system flush logs; with (select uuid from system.tables where database = currentDatabase() and table = 'data_02491') as table_uuid_ -select event_type, merge_reason from system.part_log +select + table_uuid != toUUIDOrDefault(Null), + event_type, + merge_reason, + part_name +from system.part_log where database = currentDatabase() and table = 'data_02491' and - table_uuid = table_uuid_ and - table_uuid != toUUIDOrDefault(Null) + table_uuid = table_uuid_ order by event_time_microseconds; drop table data_02491; diff --git a/tests/queries/0_stateless/02493_analyzer_sum_if_to_count_if.reference b/tests/queries/0_stateless/02493_analyzer_sum_if_to_count_if.reference new file mode 100644 index 00000000000..eccf51501ed --- /dev/null +++ b/tests/queries/0_stateless/02493_analyzer_sum_if_to_count_if.reference @@ -0,0 +1,77 @@ +QUERY id: 0 + PROJECTION COLUMNS + sumIf(1, equals(modulo(number, 2), 0)) UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: countIf, function_type: aggregate, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 1 + FUNCTION id: 4, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: modulo, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9 + CONSTANT id: 10, constant_value: UInt64_2, constant_value_type: UInt8 + CONSTANT id: 11, constant_value: UInt64_0, constant_value_type: UInt8 + JOIN TREE + TABLE_FUNCTION id: 9, table_function_name: numbers + ARGUMENTS + LIST id: 12, nodes: 1 + CONSTANT id: 13, constant_value: UInt64_10, constant_value_type: UInt8 +-- +5 +-- +QUERY id: 0 + PROJECTION COLUMNS + sum(if(equals(modulo(number, 2), 0), 1, 0)) UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: countIf, function_type: aggregate, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 1 + FUNCTION id: 4, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: modulo, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9 + CONSTANT id: 10, constant_value: UInt64_2, constant_value_type: UInt8 + CONSTANT id: 11, constant_value: UInt64_0, constant_value_type: UInt8 + JOIN TREE + TABLE_FUNCTION id: 9, table_function_name: numbers + ARGUMENTS + LIST id: 12, nodes: 1 + CONSTANT id: 13, constant_value: UInt64_10, constant_value_type: UInt8 +-- +5 +-- +QUERY id: 0 + PROJECTION COLUMNS + sum(if(equals(modulo(number, 2), 0), 0, 1)) UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: countIf, function_type: aggregate, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 1 + FUNCTION id: 4, function_name: not, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 1 + FUNCTION id: 6, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + FUNCTION id: 8, function_name: modulo, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 9, nodes: 2 + COLUMN id: 10, column_name: number, result_type: UInt64, source_id: 11 + CONSTANT id: 12, constant_value: UInt64_2, constant_value_type: UInt8 + CONSTANT id: 13, constant_value: UInt64_0, constant_value_type: UInt8 + JOIN TREE + TABLE_FUNCTION id: 11, table_function_name: numbers + ARGUMENTS + LIST id: 14, nodes: 1 + CONSTANT id: 15, constant_value: UInt64_10, constant_value_type: UInt8 +-- +5 diff --git a/tests/queries/0_stateless/02493_analyzer_sum_if_to_count_if.sql b/tests/queries/0_stateless/02493_analyzer_sum_if_to_count_if.sql new file mode 100644 index 00000000000..f1dbfa1f32a --- /dev/null +++ b/tests/queries/0_stateless/02493_analyzer_sum_if_to_count_if.sql @@ -0,0 +1,24 @@ +SET allow_experimental_analyzer = 1; +SET optimize_rewrite_sum_if_to_count_if = 1; + +EXPLAIN QUERY TREE (SELECT sumIf(1, (number % 2) == 0) FROM numbers(10)); + +SELECT '--'; + +SELECT sumIf(1, (number % 2) == 0) FROM numbers(10); + +SELECT '--'; + +EXPLAIN QUERY TREE (SELECT sum(if((number % 2) == 0, 1, 0)) FROM numbers(10)); + +SELECT '--'; + +SELECT sum(if((number % 2) == 0, 1, 0)) FROM numbers(10); + +SELECT '--'; + +EXPLAIN QUERY TREE (SELECT sum(if((number % 2) == 0, 0, 1)) FROM numbers(10)); + +SELECT '--'; + +SELECT sum(if((number % 2) == 0, 0, 1)) FROM numbers(10); diff --git a/tests/queries/0_stateless/02493_analyzer_table_functions_untuple.reference b/tests/queries/0_stateless/02493_analyzer_table_functions_untuple.reference new file mode 100644 index 00000000000..c9a8d73701d --- /dev/null +++ b/tests/queries/0_stateless/02493_analyzer_table_functions_untuple.reference @@ -0,0 +1,28 @@ +0 +-- +0 +1 +-- +1 +2 +-- +(1) 0 +-- +(0,1) 0 +-- +(1,2) 1 +(1,2) 2 +-- +(1) 0 +-- +(0,1) 0 +-- +(1,2) 1 +(1,2) 2 +-- +('1') 0 +-- +('0','1') 0 +-- +('1','2') 1 +('1','2') 2 diff --git a/tests/queries/0_stateless/02493_analyzer_table_functions_untuple.sql b/tests/queries/0_stateless/02493_analyzer_table_functions_untuple.sql new file mode 100644 index 00000000000..bdbe65c643b --- /dev/null +++ b/tests/queries/0_stateless/02493_analyzer_table_functions_untuple.sql @@ -0,0 +1,47 @@ +SET allow_experimental_analyzer = 1; + +SELECT number FROM numbers(untuple(tuple(1))); + +SELECT '--'; + +SELECT number FROM numbers(untuple(tuple(0, 2))); + +SELECT '--'; + +SELECT number FROM numbers(untuple(tuple(1, 2))); + +SELECT '--'; + +SELECT cast(tuple(1), 'Tuple(value UInt64)') AS value, number FROM numbers(untuple(value)); + +SELECT '--'; + +SELECT cast(tuple(0, 1), 'Tuple(value_1 UInt64, value_2 UInt64)') AS value, number FROM numbers(untuple(value)); + +SELECT '--'; + +SELECT cast(tuple(1, 2), 'Tuple(value_1 UInt64, value_2 UInt64)') AS value, number FROM numbers(untuple(value)); + +SELECT '--'; + +SELECT cast(tuple(1), 'Tuple(value UInt64)') AS value, number FROM numbers(value.*); + +SELECT '--'; + +SELECT cast(tuple(0, 1), 'Tuple(value_1 UInt64, value_2 UInt64)') AS value, number FROM numbers(value.*); + +SELECT '--'; + +SELECT cast(tuple(1, 2), 'Tuple(value_1 UInt64, value_2 UInt64)') AS value, number FROM numbers(value.*); + +SELECT '--'; + +SELECT cast(tuple('1'), 'Tuple(value String)') AS value, number FROM numbers(value.* APPLY x -> toUInt64(x)); + +SELECT '--'; + +SELECT cast(tuple('0', '1'), 'Tuple(value_1 String, value_2 String)') AS value, number FROM numbers(value.* APPLY x -> toUInt64(x)); + +SELECT '--'; + +SELECT cast(tuple('1', '2'), 'Tuple(value_1 String, value_2 String)') AS value, number FROM numbers(value.* APPLY x -> toUInt64(x)); diff --git a/tests/queries/0_stateless/02493_analyzer_uniq_injective_functions_elimination.reference b/tests/queries/0_stateless/02493_analyzer_uniq_injective_functions_elimination.reference new file mode 100644 index 00000000000..01ba2d19950 --- /dev/null +++ b/tests/queries/0_stateless/02493_analyzer_uniq_injective_functions_elimination.reference @@ -0,0 +1,20 @@ +QUERY id: 0 + PROJECTION COLUMNS + uniqCombined(tuple(\'\')) UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: uniqCombined, function_type: aggregate, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 1 + CONSTANT id: 4, constant_value: Tuple_(\'\'), constant_value_type: Tuple(String) + EXPRESSION + FUNCTION id: 5, function_name: tuple, function_type: ordinary, result_type: Tuple(String) + ARGUMENTS + LIST id: 6, nodes: 1 + CONSTANT id: 7, constant_value: \'\', constant_value_type: String + JOIN TREE + TABLE_FUNCTION id: 8, table_function_name: numbers + ARGUMENTS + LIST id: 9, nodes: 1 + CONSTANT id: 10, constant_value: UInt64_1, constant_value_type: UInt8 +1 diff --git a/tests/queries/0_stateless/02493_analyzer_uniq_injective_functions_elimination.sql b/tests/queries/0_stateless/02493_analyzer_uniq_injective_functions_elimination.sql new file mode 100644 index 00000000000..830db274678 --- /dev/null +++ b/tests/queries/0_stateless/02493_analyzer_uniq_injective_functions_elimination.sql @@ -0,0 +1,5 @@ +SET allow_experimental_analyzer = 1; + +EXPLAIN QUERY TREE SELECT uniqCombined(tuple('')) FROM numbers(1); + +SELECT uniqCombined(tuple('')) FROM numbers(1); diff --git a/tests/queries/0_stateless/02493_do_not_assume_that_the_original_query_was_valid_when_transforming_joins.reference b/tests/queries/0_stateless/02493_do_not_assume_that_the_original_query_was_valid_when_transforming_joins.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02493_do_not_assume_that_the_original_query_was_valid_when_transforming_joins.sql b/tests/queries/0_stateless/02493_do_not_assume_that_the_original_query_was_valid_when_transforming_joins.sql new file mode 100644 index 00000000000..6df5623638d --- /dev/null +++ b/tests/queries/0_stateless/02493_do_not_assume_that_the_original_query_was_valid_when_transforming_joins.sql @@ -0,0 +1,26 @@ +CREATE TABLE table1 (column1 String) ENGINE=MergeTree() ORDER BY tuple(); +CREATE TABLE table2 (column1 String, column2 String, column3 String) ENGINE=MergeTree() ORDER BY tuple(); +CREATE TABLE table3 (column3 String) ENGINE=MergeTree() ORDER BY tuple(); + +SELECT + * +FROM +( + SELECT + column1 + FROM table1 + GROUP BY + column1 +) AS a +ANY LEFT JOIN +( + SELECT + * + FROM table2 +) AS b ON (b.column1 = a.column1) AND (b.column2 = a.column2) +ANY LEFT JOIN +( + SELECT + * + FROM table3 +) AS c ON c.column3 = b.column3; -- {serverError UNKNOWN_IDENTIFIER} diff --git a/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.reference b/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.reference new file mode 100644 index 00000000000..f517be778ed --- /dev/null +++ b/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.reference @@ -0,0 +1,41 @@ +-- { echo } + +-- The number of output streams is limited by max_streams_for_merge_tree_reading +select sum(x) from t settings max_threads=32, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=0; +49999995000000 +select * from (explain pipeline select sum(x) from t settings max_threads=32, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=0) where explain like '%Resize%' or explain like '%MergeTreeThread%'; + Resize 16 → 32 + StrictResize 16 → 16 + MergeTreeThread × 16 0 → 1 +-- Without asynchronous_read, max_streams_for_merge_tree_reading limits max_streams * max_streams_to_max_threads_ratio +select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=0, max_streams_to_max_threads_ratio=8; +49999995000000 +select * from (explain pipeline select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=0, max_streams_to_max_threads_ratio=8) where explain like '%Resize%' or explain like '%MergeTreeThread%'; + Resize 16 → 4 + StrictResize 16 → 16 + MergeTreeThread × 16 0 → 1 +-- With asynchronous_read, read in max_streams_for_merge_tree_reading async streams and resize to max_threads +select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1; +49999995000000 +select * from (explain pipeline select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1) where explain like '%Resize%' or explain like '%MergeTreeThread%'; + Resize 4 → 4 + StrictResize 4 → 4 + Resize 16 → 4 + MergeTreeThread × 16 0 → 1 +-- With asynchronous_read, read using max_streams * max_streams_to_max_threads_ratio async streams, resize to max_streams_for_merge_tree_reading outp[ut streams, resize to max_threads after aggregation +select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, max_streams_to_max_threads_ratio=8; +49999995000000 +select * from (explain pipeline select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, max_streams_to_max_threads_ratio=8) where explain like '%Resize%' or explain like '%MergeTreeThread%'; + Resize 16 → 4 + StrictResize 16 → 16 + Resize 32 → 16 + MergeTreeThread × 32 0 → 1 +-- For read-in-order, disable everything +select sum(x) from (select x from t order by x) settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, optimize_read_in_order=1, query_plan_read_in_order=1; +49999995000000 +select * from (explain pipeline select sum(x) from (select x from t order by x) settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, optimize_read_in_order=1, query_plan_read_in_order=1) where explain like '%Resize%'; + Resize 1 → 4 +select sum(x) from (select x from t order by x) settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, max_streams_to_max_threads_ratio=8, optimize_read_in_order=1, query_plan_read_in_order=1; +49999995000000 +select * from (explain pipeline select sum(x) from (select x from t order by x) settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, max_streams_to_max_threads_ratio=8, optimize_read_in_order=1, query_plan_read_in_order=1) where explain like '%Resize%'; + Resize 1 → 4 diff --git a/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql b/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql new file mode 100644 index 00000000000..29fb6062a8e --- /dev/null +++ b/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql @@ -0,0 +1,26 @@ +create table t (x UInt64) engine = MergeTree order by x; +insert into t select number from numbers_mt(10000000) settings max_insert_threads=8; + +-- { echo } + +-- The number of output streams is limited by max_streams_for_merge_tree_reading +select sum(x) from t settings max_threads=32, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=0; +select * from (explain pipeline select sum(x) from t settings max_threads=32, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=0) where explain like '%Resize%' or explain like '%MergeTreeThread%'; + +-- Without asynchronous_read, max_streams_for_merge_tree_reading limits max_streams * max_streams_to_max_threads_ratio +select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=0, max_streams_to_max_threads_ratio=8; +select * from (explain pipeline select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=0, max_streams_to_max_threads_ratio=8) where explain like '%Resize%' or explain like '%MergeTreeThread%'; + +-- With asynchronous_read, read in max_streams_for_merge_tree_reading async streams and resize to max_threads +select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1; +select * from (explain pipeline select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1) where explain like '%Resize%' or explain like '%MergeTreeThread%'; + +-- With asynchronous_read, read using max_streams * max_streams_to_max_threads_ratio async streams, resize to max_streams_for_merge_tree_reading outp[ut streams, resize to max_threads after aggregation +select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, max_streams_to_max_threads_ratio=8; +select * from (explain pipeline select sum(x) from t settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, max_streams_to_max_threads_ratio=8) where explain like '%Resize%' or explain like '%MergeTreeThread%'; + +-- For read-in-order, disable everything +select sum(x) from (select x from t order by x) settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, optimize_read_in_order=1, query_plan_read_in_order=1; +select * from (explain pipeline select sum(x) from (select x from t order by x) settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, optimize_read_in_order=1, query_plan_read_in_order=1) where explain like '%Resize%'; +select sum(x) from (select x from t order by x) settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, max_streams_to_max_threads_ratio=8, optimize_read_in_order=1, query_plan_read_in_order=1; +select * from (explain pipeline select sum(x) from (select x from t order by x) settings max_threads=4, max_streams_for_merge_tree_reading=16, allow_asynchronous_read_from_io_pool_for_merge_tree=1, max_streams_to_max_threads_ratio=8, optimize_read_in_order=1, query_plan_read_in_order=1) where explain like '%Resize%'; diff --git a/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.reference b/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.reference new file mode 100644 index 00000000000..42feff405c0 --- /dev/null +++ b/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.reference @@ -0,0 +1 @@ +[[1]] diff --git a/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql b/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql new file mode 100644 index 00000000000..2af556ce9ab --- /dev/null +++ b/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql @@ -0,0 +1,16 @@ +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table ( + fingerprint UInt16, + fields Array(Tuple(name Array(UInt32), value String)) +) ENGINE = MergeTree +ORDER BY fingerprint; + +INSERT INTO test_table VALUES (0, [[1]], ['1']); + +SELECT fields.name FROM (SELECT fields.name FROM test_table); + +SELECT fields.name, fields.value FROM (SELECT fields.name FROM test_table); -- { serverError 36 } + +DROP TABLE IF EXISTS test_table; diff --git a/tests/queries/0_stateless/02494_optimize_group_by_function_keys_and_alias_columns.reference b/tests/queries/0_stateless/02494_optimize_group_by_function_keys_and_alias_columns.reference new file mode 100644 index 00000000000..83171ee33ec --- /dev/null +++ b/tests/queries/0_stateless/02494_optimize_group_by_function_keys_and_alias_columns.reference @@ -0,0 +1,5 @@ +20221123 2022-11-23 22:33:19 +20221124 2022-11-24 22:33:19 +20221125 2022-11-25 22:33:19 +20221126 2022-11-26 22:33:19 +20221127 2022-11-27 22:33:19 diff --git a/tests/queries/0_stateless/02494_optimize_group_by_function_keys_and_alias_columns.sql b/tests/queries/0_stateless/02494_optimize_group_by_function_keys_and_alias_columns.sql new file mode 100644 index 00000000000..ae4654bb135 --- /dev/null +++ b/tests/queries/0_stateless/02494_optimize_group_by_function_keys_and_alias_columns.sql @@ -0,0 +1,7 @@ +CREATE TABLE t(timestamp DateTime, day ALIAS toYYYYMMDD(timestamp)) Engine = MergeTree ORDER BY timestamp; + +INSERT INTO t (timestamp) VALUES ('2022-11-25 22:33:19'::DateTime), ('2022-11-25 22:33:19'::DateTime - INTERVAL 1 DAY), ('2022-11-25 22:33:19'::DateTime + INTERVAL 1 DAY), ('2022-11-25 22:33:19'::DateTime - INTERVAL 2 DAY), ('2022-11-25 22:33:19'::DateTime + INTERVAL 2 DAY); +INSERT INTO t (timestamp) VALUES ('2022-11-25 22:33:19'::DateTime), ('2022-11-25 22:33:19'::DateTime - INTERVAL 1 DAY), ('2022-11-25 22:33:19'::DateTime + INTERVAL 1 DAY), ('2022-11-25 22:33:19'::DateTime - INTERVAL 2 DAY), ('2022-11-25 22:33:19'::DateTime + INTERVAL 2 DAY); +INSERT INTO t (timestamp) VALUES ('2022-11-25 22:33:19'::DateTime), ('2022-11-25 22:33:19'::DateTime - INTERVAL 1 DAY), ('2022-11-25 22:33:19'::DateTime + INTERVAL 1 DAY), ('2022-11-25 22:33:19'::DateTime - INTERVAL 2 DAY), ('2022-11-25 22:33:19'::DateTime + INTERVAL 2 DAY); + +SELECT day, timestamp FROM remote('127.0.0.{1,2}', currentDatabase(), t) GROUP BY day, timestamp ORDER BY timestamp; diff --git a/tests/queries/0_stateless/02494_trace_log_profile_events.reference b/tests/queries/0_stateless/02494_trace_log_profile_events.reference new file mode 100644 index 00000000000..cd121fd3feb --- /dev/null +++ b/tests/queries/0_stateless/02494_trace_log_profile_events.reference @@ -0,0 +1,2 @@ +1 +1 1 diff --git a/tests/queries/0_stateless/02494_trace_log_profile_events.sh b/tests/queries/0_stateless/02494_trace_log_profile_events.sh new file mode 100755 index 00000000000..4dd0a34d202 --- /dev/null +++ b/tests/queries/0_stateless/02494_trace_log_profile_events.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Tags: no-tsan, no-parallel + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +query_id="$RANDOM-$CLICKHOUSE_DATABASE" +${CLICKHOUSE_CLIENT} --query_id $query_id --query "SELECT 1 FORMAT Null SETTINGS trace_profile_events = 0" + +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" +${CLICKHOUSE_CLIENT} --query "SELECT count() = 0 FROM system.trace_log WHERE query_id = '$query_id' AND trace_type = 'ProfileEvent'" + +query_id="$RANDOM-$CLICKHOUSE_DATABASE" +${CLICKHOUSE_CLIENT} --query_id $query_id --query "SELECT 1 FORMAT Null SETTINGS trace_profile_events = 1" + +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" +${CLICKHOUSE_CLIENT} --query "SELECT count() > 0, sum(empty(trace)) = 0 FROM system.trace_log WHERE query_id = '$query_id' AND trace_type = 'ProfileEvent'" diff --git a/tests/queries/0_stateless/02494_zero_copy_and_projection_and_mutation_work_together.reference b/tests/queries/0_stateless/02494_zero_copy_and_projection_and_mutation_work_together.reference new file mode 100644 index 00000000000..726e74146fc --- /dev/null +++ b/tests/queries/0_stateless/02494_zero_copy_and_projection_and_mutation_work_together.reference @@ -0,0 +1,4 @@ +199 +199 +1990 199 +1990 199 diff --git a/tests/queries/0_stateless/02494_zero_copy_and_projection_and_mutation_work_together.sql b/tests/queries/0_stateless/02494_zero_copy_and_projection_and_mutation_work_together.sql new file mode 100644 index 00000000000..7a51d86dd30 --- /dev/null +++ b/tests/queries/0_stateless/02494_zero_copy_and_projection_and_mutation_work_together.sql @@ -0,0 +1,79 @@ +DROP TABLE IF EXISTS wikistat1; +DROP TABLE IF EXISTS wikistat2; + +CREATE TABLE wikistat1 +( + time DateTime, + project LowCardinality(String), + subproject LowCardinality(String), + path String, + hits UInt64, + PROJECTION total + ( + SELECT + project, + subproject, + path, + sum(hits), + count() + GROUP BY + project, + subproject, + path + ) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/02494_zero_copy_and_projection', '1') +ORDER BY (path, time) +SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0, allow_remote_fs_zero_copy_replication=1, min_bytes_for_wide_part=0; + +CREATE TABLE wikistat2 +( + time DateTime, + project LowCardinality(String), + subproject LowCardinality(String), + path String, + hits UInt64, + PROJECTION total + ( + SELECT + project, + subproject, + path, + sum(hits), + count() + GROUP BY + project, + subproject, + path + ) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/02494_zero_copy_and_projection', '2') +ORDER BY (path, time) +SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0, allow_remote_fs_zero_copy_replication=1, min_bytes_for_wide_part=0; + +INSERT INTO wikistat1 SELECT toDateTime('2020-10-01 00:00:00'), 'hello', 'world', '/data/path', 10 from numbers(100); + +INSERT INTO wikistat1 SELECT toDateTime('2020-10-01 00:00:00'), 'hello', 'world', '/data/path', 10 from numbers(99, 99); + +SYSTEM SYNC REPLICA wikistat2; + +SELECT COUNT() from wikistat1 WHERE NOT ignore(*); +SELECT COUNT() from wikistat2 WHERE NOT ignore(*); + +SYSTEM STOP REPLICATION QUEUES wikistat2; + +ALTER TABLE wikistat1 DELETE where time = toDateTime('2022-12-20 00:00:00') SETTINGS mutations_sync = 1; + +SYSTEM START REPLICATION QUEUES wikistat2; + +SYSTEM SYNC REPLICA wikistat2; + +-- it doesn't make test flaky, rarely we will not delete the parts because of cleanup thread was slow. +-- Such condition will lead to successful queries. +SELECT 0 FROM numbers(5) WHERE sleepEachRow(1) = 1; + +select sum(hits), count() from wikistat1 GROUP BY project, subproject, path settings allow_experimental_projection_optimization = 1, force_optimize_projection = 1; +select sum(hits), count() from wikistat2 GROUP BY project, subproject, path settings allow_experimental_projection_optimization = 1, force_optimize_projection = 1; + +DROP TABLE wikistat1; +DROP TABLE wikistat2; diff --git a/tests/queries/0_stateless/02496_row_binary_large_string_size.reference b/tests/queries/0_stateless/02496_row_binary_large_string_size.reference new file mode 100644 index 00000000000..d86bac9de59 --- /dev/null +++ b/tests/queries/0_stateless/02496_row_binary_large_string_size.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/02496_row_binary_large_string_size.sh b/tests/queries/0_stateless/02496_row_binary_large_string_size.sh new file mode 100755 index 00000000000..39f83f6c2b8 --- /dev/null +++ b/tests/queries/0_stateless/02496_row_binary_large_string_size.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +printf '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' | $CLICKHOUSE_LOCAL --format_binary_max_string_size=100000 --input-format=RowBinary --structure='s String' -q "select * from table" 2>&1 | grep -q -F "TOO_LARGE_STRING_SIZE" && echo "OK" || echo FAIL"" diff --git a/tests/queries/0_stateless/data_parquet/int-list-zero-based-chunked-array.parquet b/tests/queries/0_stateless/data_parquet/int-list-zero-based-chunked-array.parquet new file mode 100644 index 00000000000..2eb3ba3ab15 Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/int-list-zero-based-chunked-array.parquet differ diff --git a/tests/queries/0_stateless/data_parquet/list_monotonically_increasing_offsets.parquet b/tests/queries/0_stateless/data_parquet/list_monotonically_increasing_offsets.parquet new file mode 100644 index 00000000000..1c23e27db65 Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/list_monotonically_increasing_offsets.parquet differ diff --git a/tests/queries/0_stateless/filesystem_cache_queries/02240_system_filesystem_cache_table.queries b/tests/queries/0_stateless/filesystem_cache_queries/02240_system_filesystem_cache_table.queries index ab73e97b96e..228dccfcb5b 100644 --- a/tests/queries/0_stateless/filesystem_cache_queries/02240_system_filesystem_cache_table.queries +++ b/tests/queries/0_stateless/filesystem_cache_queries/02240_system_filesystem_cache_table.queries @@ -3,7 +3,7 @@ SYSTEM DROP FILESYSTEM CACHE; SET enable_filesystem_cache_on_write_operations=0; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='_storagePolicy', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='_storagePolicy', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; INSERT INTO test SELECT number, toString(number) FROM numbers(100); @@ -18,7 +18,7 @@ SYSTEM DROP FILESYSTEM CACHE; SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='_storagePolicy_3', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='_storagePolicy_3', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; INSERT INTO test SELECT number, toString(number) FROM numbers(100); SELECT * FROM test FORMAT Null; SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_end, size; diff --git a/tests/queries/0_stateless/filesystem_cache_queries/02241_filesystem_cache_on_write_operations.queries b/tests/queries/0_stateless/filesystem_cache_queries/02241_filesystem_cache_on_write_operations.queries index 76aebfcaca3..bd185942e6c 100644 --- a/tests/queries/0_stateless/filesystem_cache_queries/02241_filesystem_cache_on_write_operations.queries +++ b/tests/queries/0_stateless/filesystem_cache_queries/02241_filesystem_cache_on_write_operations.queries @@ -3,7 +3,7 @@ SET enable_filesystem_cache_on_write_operations=1; DROP TABLE IF EXISTS test; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='_storagePolicy', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='_storagePolicy', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; SYSTEM DROP FILESYSTEM CACHE; SELECT file_segment_range_begin, file_segment_range_end, size, state diff --git a/tests/queries/0_stateless/filesystem_cache_queries/02242_system_filesystem_cache_log_table.queries b/tests/queries/0_stateless/filesystem_cache_queries/02242_system_filesystem_cache_log_table.queries index 386a1792ea4..56a8710cc93 100644 --- a/tests/queries/0_stateless/filesystem_cache_queries/02242_system_filesystem_cache_log_table.queries +++ b/tests/queries/0_stateless/filesystem_cache_queries/02242_system_filesystem_cache_log_table.queries @@ -6,7 +6,7 @@ SET enable_filesystem_cache_on_write_operations=0; DROP TABLE IF EXISTS test; DROP TABLE IF EXISTS system.filesystem_cache_log; -CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='_storagePolicy', min_bytes_for_wide_part = 10485760; +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='_storagePolicy', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false; SYSTEM STOP MERGES test; INSERT INTO test SELECT number, toString(number) FROM numbers(100000); diff --git a/tests/queries/0_stateless/parts.lib b/tests/queries/0_stateless/parts.lib new file mode 100644 index 00000000000..c35f996ffed --- /dev/null +++ b/tests/queries/0_stateless/parts.lib @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +function wait_for_delete_empty_parts() +{ + local table=$1 + local database=${2:-$CLICKHOUSE_DATABASE} + local timeout=${3:-20} + + while [[ timeout -gt 0 ]] + do + res=$(${CLICKHOUSE_CLIENT} --query="SELECT count() FROM system.parts WHERE database='$database' AND table='$table' AND active AND rows=0") + [[ $res -eq 0 ]] && return 0 + + sleep 2 + timeout=$((timeout - 2)) + done + + echo "Timed out while waiting for delete empty parts!" >&2 + return 2 +} + +function wait_for_delete_inactive_parts() +{ + local table=$1 + local database=${2:-$CLICKHOUSE_DATABASE} + local timeout=${3:-20} + + while [[ timeout -gt 0 ]] + do + res=$(${CLICKHOUSE_CLIENT} --query="SELECT count() FROM system.parts WHERE database='$database' AND table='$table' AND not active") + [[ $res -eq 0 ]] && return 0 + + sleep 2 + timeout=$((timeout - 2)) + done + + echo "Timed out while waiting for delete inactive parts!" >&2 + return 2 +} diff --git a/tests/queries/1_stateful/00152_insert_different_granularity.sql b/tests/queries/1_stateful/00152_insert_different_granularity.sql index 6415cdad8a5..294d71b384b 100644 --- a/tests/queries/1_stateful/00152_insert_different_granularity.sql +++ b/tests/queries/1_stateful/00152_insert_different_granularity.sql @@ -32,7 +32,12 @@ ALTER TABLE test.hits ATTACH PARTITION 201403; DROP TABLE IF EXISTS hits_copy; -CREATE TABLE hits_copy (`WatchID` UInt64, `JavaEnable` UInt8, `Title` String, `GoodEvent` Int16, `EventTime` DateTime, `EventDate` Date, `CounterID` UInt32, `ClientIP` UInt32, `ClientIP6` FixedString(16), `RegionID` UInt32, `UserID` UInt64, `CounterClass` Int8, `OS` UInt8, `UserAgent` UInt8, `URL` String, `Referer` String, `URLDomain` String, `RefererDomain` String, `Refresh` UInt8, `IsRobot` UInt8, `RefererCategories` Array(UInt16), `URLCategories` Array(UInt16), `URLRegions` Array(UInt32), `RefererRegions` Array(UInt32), `ResolutionWidth` UInt16, `ResolutionHeight` UInt16, `ResolutionDepth` UInt8, `FlashMajor` UInt8, `FlashMinor` UInt8, `FlashMinor2` String, `NetMajor` UInt8, `NetMinor` UInt8, `UserAgentMajor` UInt16, `UserAgentMinor` FixedString(2), `CookieEnable` UInt8, `JavascriptEnable` UInt8, `IsMobile` UInt8, `MobilePhone` UInt8, `MobilePhoneModel` String, `Params` String, `IPNetworkID` UInt32, `TraficSourceID` Int8, `SearchEngineID` UInt16, `SearchPhrase` String, `AdvEngineID` UInt8, `IsArtifical` UInt8, `WindowClientWidth` UInt16, `WindowClientHeight` UInt16, `ClientTimeZone` Int16, `ClientEventTime` DateTime, `SilverlightVersion1` UInt8, `SilverlightVersion2` UInt8, `SilverlightVersion3` UInt32, `SilverlightVersion4` UInt16, `PageCharset` String, `CodeVersion` UInt32, `IsLink` UInt8, `IsDownload` UInt8, `IsNotBounce` UInt8, `FUniqID` UInt64, `HID` UInt32, `IsOldCounter` UInt8, `IsEvent` UInt8, `IsParameter` UInt8, `DontCountHits` UInt8, `WithHash` UInt8, `HitColor` FixedString(1), `UTCEventTime` DateTime, `Age` UInt8, `Sex` UInt8, `Income` UInt8, `Interests` UInt16, `Robotness` UInt8, `GeneralInterests` Array(UInt16), `RemoteIP` UInt32, `RemoteIP6` FixedString(16), `WindowName` Int32, `OpenerName` Int32, `HistoryLength` Int16, `BrowserLanguage` FixedString(2), `BrowserCountry` FixedString(2), `SocialNetwork` String, `SocialAction` String, `HTTPError` UInt16, `SendTiming` Int32, `DNSTiming` Int32, `ConnectTiming` Int32, `ResponseStartTiming` Int32, `ResponseEndTiming` Int32, `FetchTiming` Int32, `RedirectTiming` Int32, `DOMInteractiveTiming` Int32, `DOMContentLoadedTiming` Int32, `DOMCompleteTiming` Int32, `LoadEventStartTiming` Int32, `LoadEventEndTiming` Int32, `NSToDOMContentLoadedTiming` Int32, `FirstPaintTiming` Int32, `RedirectCount` Int8, `SocialSourceNetworkID` UInt8, `SocialSourcePage` String, `ParamPrice` Int64, `ParamOrderID` String, `ParamCurrency` FixedString(3), `ParamCurrencyID` UInt16, `GoalsReached` Array(UInt32), `OpenstatServiceName` String, `OpenstatCampaignID` String, `OpenstatAdID` String, `OpenstatSourceID` String, `UTMSource` String, `UTMMedium` String, `UTMCampaign` String, `UTMContent` String, `UTMTerm` String, `FromTag` String, `HasGCLID` UInt8, `RefererHash` UInt64, `URLHash` UInt64, `CLID` UInt32, `YCLID` UInt64, `ShareService` String, `ShareURL` String, `ShareTitle` String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `IslandID` FixedString(16), `RequestNum` UInt32, `RequestTry` UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192, index_granularity_bytes=0, min_bytes_for_wide_part = 0; +CREATE TABLE hits_copy (`WatchID` UInt64, `JavaEnable` UInt8, `Title` String, `GoodEvent` Int16, `EventTime` DateTime, `EventDate` Date, `CounterID` UInt32, `ClientIP` UInt32, `ClientIP6` FixedString(16), `RegionID` UInt32, `UserID` UInt64, `CounterClass` Int8, `OS` UInt8, `UserAgent` UInt8, `URL` String, `Referer` String, `URLDomain` String, `RefererDomain` String, `Refresh` UInt8, `IsRobot` UInt8, `RefererCategories` Array(UInt16), `URLCategories` Array(UInt16), `URLRegions` Array(UInt32), `RefererRegions` Array(UInt32), `ResolutionWidth` UInt16, `ResolutionHeight` UInt16, `ResolutionDepth` UInt8, `FlashMajor` UInt8, `FlashMinor` UInt8, `FlashMinor2` String, `NetMajor` UInt8, `NetMinor` UInt8, `UserAgentMajor` UInt16, `UserAgentMinor` FixedString(2), `CookieEnable` UInt8, `JavascriptEnable` UInt8, `IsMobile` UInt8, `MobilePhone` UInt8, `MobilePhoneModel` String, `Params` String, `IPNetworkID` UInt32, `TraficSourceID` Int8, `SearchEngineID` UInt16, `SearchPhrase` String, `AdvEngineID` UInt8, `IsArtifical` UInt8, `WindowClientWidth` UInt16, `WindowClientHeight` UInt16, `ClientTimeZone` Int16, `ClientEventTime` DateTime, `SilverlightVersion1` UInt8, `SilverlightVersion2` UInt8, `SilverlightVersion3` UInt32, `SilverlightVersion4` UInt16, `PageCharset` String, `CodeVersion` UInt32, `IsLink` UInt8, `IsDownload` UInt8, `IsNotBounce` UInt8, `FUniqID` UInt64, `HID` UInt32, `IsOldCounter` UInt8, `IsEvent` UInt8, `IsParameter` UInt8, `DontCountHits` UInt8, `WithHash` UInt8, `HitColor` FixedString(1), `UTCEventTime` DateTime, `Age` UInt8, `Sex` UInt8, `Income` UInt8, `Interests` UInt16, `Robotness` UInt8, `GeneralInterests` Array(UInt16), `RemoteIP` UInt32, `RemoteIP6` FixedString(16), `WindowName` Int32, `OpenerName` Int32, `HistoryLength` Int16, `BrowserLanguage` FixedString(2), `BrowserCountry` FixedString(2), `SocialNetwork` String, `SocialAction` String, `HTTPError` UInt16, `SendTiming` Int32, `DNSTiming` Int32, `ConnectTiming` Int32, `ResponseStartTiming` Int32, `ResponseEndTiming` Int32, `FetchTiming` Int32, `RedirectTiming` Int32, `DOMInteractiveTiming` Int32, `DOMContentLoadedTiming` Int32, `DOMCompleteTiming` Int32, `LoadEventStartTiming` Int32, `LoadEventEndTiming` Int32, `NSToDOMContentLoadedTiming` Int32, `FirstPaintTiming` Int32, `RedirectCount` Int8, `SocialSourceNetworkID` UInt8, `SocialSourcePage` String, `ParamPrice` Int64, `ParamOrderID` String, `ParamCurrency` FixedString(3), `ParamCurrencyID` UInt16, `GoalsReached` Array(UInt32), `OpenstatServiceName` String, `OpenstatCampaignID` String, `OpenstatAdID` String, `OpenstatSourceID` String, `UTMSource` String, `UTMMedium` String, `UTMCampaign` String, `UTMContent` String, `UTMTerm` String, `FromTag` String, `HasGCLID` UInt8, `RefererHash` UInt64, `URLHash` UInt64, `CLID` UInt32, `YCLID` UInt64, `ShareService` String, `ShareURL` String, `ShareTitle` String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `IslandID` FixedString(16), `RequestNum` UInt32, `RequestTry` UInt8) + ENGINE = MergeTree() + PARTITION BY toYYYYMM(EventDate) + ORDER BY (CounterID, EventDate, intHash32(UserID)) + SAMPLE BY intHash32(UserID) + SETTINGS index_granularity=8192, min_bytes_for_wide_part = 0; ALTER TABLE hits_copy REPLACE PARTITION 201403 FROM test.hits; diff --git a/tests/queries/1_stateful/00172_parallel_join.reference.j2 b/tests/queries/1_stateful/00172_parallel_join.reference.j2 index 30088c91500..1a43f1fb6ef 100644 --- a/tests/queries/1_stateful/00172_parallel_join.reference.j2 +++ b/tests/queries/1_stateful/00172_parallel_join.reference.j2 @@ -1,4 +1,4 @@ -{% for join_algorithm in ['hash', 'parallel_hash', 'full_sorting_merge'] -%} +{% for join_algorithm in ['hash', 'parallel_hash', 'full_sorting_merge', 'grace_hash'] -%} --- {{ join_algorithm }} --- 2014-03-17 1406958 265108 2014-03-19 1405797 261624 @@ -24,7 +24,7 @@ mail.ru 87949 22225 best.ru 58537 55 korablitz.ru 51844 0 hurpass.com 49671 1251 -{% if join_algorithm != 'full_sorting_merge' -%} +{% if join_algorithm not in ['full_sorting_merge', 'grace_hash'] -%} 37292 0 35642 92887 252214 0 7842 196036 0 diff --git a/tests/queries/1_stateful/00172_parallel_join.sql.j2 b/tests/queries/1_stateful/00172_parallel_join.sql.j2 index 39c981e0d31..ff077f43874 100644 --- a/tests/queries/1_stateful/00172_parallel_join.sql.j2 +++ b/tests/queries/1_stateful/00172_parallel_join.sql.j2 @@ -1,4 +1,6 @@ -{% for join_algorithm in ['hash', 'parallel_hash', 'full_sorting_merge'] -%} +{% for join_algorithm in ['hash', 'parallel_hash', 'full_sorting_merge', 'grace_hash'] -%} + +SET max_bytes_in_join = '{% if join_algorithm == 'grace_hash' %}20K{% else %}0{% endif %}'; SELECT '--- {{ join_algorithm }} ---'; @@ -69,7 +71,7 @@ ORDER BY hits DESC LIMIT 10 SETTINGS joined_subquery_requires_alias = 0; -{% if join_algorithm != 'full_sorting_merge' -%} +{% if join_algorithm not in ['full_sorting_merge', 'grace_hash'] -%} SELECT CounterID FROM test.visits ARRAY JOIN Goals.ID WHERE CounterID = 942285 ORDER BY CounterID; @@ -211,4 +213,6 @@ ALL INNER JOIN ) AS b USING k ORDER BY joined; -{% endfor %} +SET max_bytes_in_join = 0; + +{% endfor -%} diff --git a/tests/queries/1_stateful/00176_bson_parallel_parsing.sh b/tests/queries/1_stateful/00176_bson_parallel_parsing.sh index 14e9ed92a01..8c021e8d3f6 100755 --- a/tests/queries/1_stateful/00176_bson_parallel_parsing.sh +++ b/tests/queries/1_stateful/00176_bson_parallel_parsing.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: disabled CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -9,14 +10,14 @@ $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_bson(WatchID UInt64, ClientIP6 Fixed $CLICKHOUSE_CLIENT --max_threads=0 --max_block_size=65505 --output_format_parallel_formatting=false -q \ -"SELECT WatchID, ClientIP6, EventTime, Title FROM test.hits ORDER BY UserID LIMIT 100000 Format BSONEachRow" > 00176_data.bson +"SELECT WatchID, ClientIP6, EventTime, Title FROM test.hits ORDER BY UserID LIMIT 30000 Format BSONEachRow" > 00176_data.bson -cat 00176_data.bson | $CLICKHOUSE_CLIENT --max_threads=0 --max_block_size=65505 --input_format_parallel_parsing=false -q "INSERT INTO parsing_bson FORMAT BSONEachRow" +cat 00176_data.bson | $CLICKHOUSE_CLIENT --max_threads=0 --input_format_parallel_parsing=false -q "INSERT INTO parsing_bson FORMAT BSONEachRow" checksum1=$($CLICKHOUSE_CLIENT -q "SELECT * FROM parsing_bson ORDER BY WatchID;" | md5sum) $CLICKHOUSE_CLIENT -q "TRUNCATE TABLE parsing_bson;" -cat 00176_data.bson | $CLICKHOUSE_CLIENT --max_threads=0 --max_block_size=65505 --input_format_parallel_parsing=true -q "INSERT INTO parsing_bson FORMAT BSONEachRow" +cat 00176_data.bson | $CLICKHOUSE_CLIENT --max_threads=0 --max_insert_block_size=5000 --input_format_parallel_parsing=true -q "INSERT INTO parsing_bson FORMAT BSONEachRow" checksum2=$($CLICKHOUSE_CLIENT -q "SELECT * FROM parsing_bson ORDER BY WatchID;" | md5sum) diff --git a/tests/queries/1_stateful/00176_distinct_limit_by_limit_bug_43377.reference b/tests/queries/1_stateful/00176_distinct_limit_by_limit_bug_43377.reference new file mode 100644 index 00000000000..f599e28b8ab --- /dev/null +++ b/tests/queries/1_stateful/00176_distinct_limit_by_limit_bug_43377.reference @@ -0,0 +1 @@ +10 diff --git a/tests/queries/1_stateful/00176_distinct_limit_by_limit_bug_43377.sql b/tests/queries/1_stateful/00176_distinct_limit_by_limit_bug_43377.sql new file mode 100644 index 00000000000..6397d7f5a28 --- /dev/null +++ b/tests/queries/1_stateful/00176_distinct_limit_by_limit_bug_43377.sql @@ -0,0 +1,11 @@ +SELECT count() +FROM +( + SELECT DISTINCT + Title, + SearchPhrase + FROM test.hits + WHERE (SearchPhrase != '') AND (NOT match(Title, '[а-яА-ЯёЁ]')) AND (NOT match(SearchPhrase, '[а-яА-ЯёЁ]')) + LIMIT 1 BY Title + LIMIT 10 +); diff --git a/utils/check-style/check-mypy b/utils/check-style/check-mypy new file mode 100755 index 00000000000..42cb7fbbd15 --- /dev/null +++ b/utils/check-style/check-mypy @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# The mypy supports pyproject.toml, but unfortunately it doesn't support it recursively +# https://github.com/python/mypy/issues/10613 +# +# Unless it's done, mypy only runs against tests/ci +# Let's leave here a room for improvement and redo it when mypy will test anything else + +GIT_ROOT=$(git rev-parse --show-cdup) +GIT_ROOT=${GIT_ROOT:-.} +CONFIG="$GIT_ROOT/tests/ci/.mypy.ini" +DIRS=("$GIT_ROOT/tests/ci/" "$GIT_ROOT/tests/ci/"*/) +tmp=$(mktemp) +for dir in "${DIRS[@]}"; do + if ! compgen -G "$dir"/*.py > /dev/null; then + continue + fi + if ! mypy --config-file="$CONFIG" --sqlite-cache "$dir"/*.py > "$tmp" 2>&1; then + echo "Errors while processing $dir": + cat "$tmp" + fi +done +rm -rf "$tmp" diff --git a/utils/check-style/codespell-ignore-words.list b/utils/check-style/codespell-ignore-words.list index f331e222541..9c26f322c8e 100644 --- a/utils/check-style/codespell-ignore-words.list +++ b/utils/check-style/codespell-ignore-words.list @@ -23,3 +23,4 @@ hastable nam ubuntu toolchain +vie