diff --git a/.clang-tidy b/.clang-tidy index 860e7b3189f..532b0f37b81 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -22,6 +22,8 @@ Checks: '*, -bugprone-implicit-widening-of-multiplication-result, -bugprone-narrowing-conversions, -bugprone-not-null-terminated-result, + -bugprone-unchecked-optional-access, + -bugprone-assignment-in-if-condition, -cert-dcl16-c, -cert-err58-cpp, @@ -103,6 +105,7 @@ Checks: '*, -misc-no-recursion, -misc-non-private-member-variables-in-classes, + -misc-const-correctness, -modernize-avoid-c-arrays, -modernize-concat-nested-namespaces, @@ -114,6 +117,7 @@ Checks: '*, -modernize-use-nodiscard, -modernize-use-override, -modernize-use-trailing-return-type, + -modernize-macro-to-enum, -performance-inefficient-string-concatenation, -performance-no-int-to-ptr, @@ -135,6 +139,7 @@ Checks: '*, -readability-suspicious-call-argument, -readability-uppercase-literal-suffix, -readability-use-anyofallof, + -readability-simplify-boolean-expr, -zirkon-*, ' diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000000..06e893fabb3 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,15 @@ +# This is a file that can be used by git-blame to ignore some revisions. +# (git 2.23+, released in August 2019) +# +# Can be configured as follow: +# +# $ git config blame.ignoreRevsFile .git-blame-ignore-revs +# +# For more information you can look at git-blame(1) man page. + +# Changed tabs to spaces in code [#CLICKHOUSE-3] +137ad95929ee016cc6d3c03bccb5586941c163ff + +# dbms/ → src/ +# (though it is unlikely that you will see it in blame) +06446b4f08a142d6f1bc30664c47ded88ab51782 diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 1c51d06f395..4c8d023f2ec 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -112,10 +112,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ######################################################################################### #################################### ORDINARY BUILDS #################################### @@ -162,10 +160,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] @@ -209,10 +205,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAsan: needs: [DockerHubPush] @@ -254,10 +248,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebTsan: needs: [DockerHubPush] @@ -299,10 +291,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebDebug: needs: [DockerHubPush] @@ -344,10 +334,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwin: needs: [DockerHubPush] @@ -391,10 +379,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwinAarch64: needs: [DockerHubPush] @@ -438,10 +424,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ############################################################################################ ##################################### Docker images ####################################### @@ -468,10 +452,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################ ##################################### BUILD REPORTER ####################################### @@ -514,10 +496,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderSpecialReport: needs: @@ -554,10 +534,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ########################### FUNCTIONAl STATELESS TESTS ####################################### @@ -594,10 +572,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### @@ -634,10 +610,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ######################################### STRESS TESTS ####################################### @@ -677,10 +651,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -716,10 +688,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FinishCheck: needs: diff --git a/.github/workflows/cherry_pick.yml b/.github/workflows/cherry_pick.yml index e6a10479c7e..3e6f9e76c56 100644 --- a/.github/workflows/cherry_pick.yml +++ b/.github/workflows/cherry_pick.yml @@ -40,8 +40,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" diff --git a/.github/workflows/docs_check.yml b/.github/workflows/docs_check.yml index b50584a2c01..7a15e77becb 100644 --- a/.github/workflows/docs_check.yml +++ b/.github/workflows/docs_check.yml @@ -125,10 +125,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" DocsCheck: needs: DockerHubPush @@ -158,10 +156,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FinishCheck: needs: diff --git a/.github/workflows/docs_release.yml b/.github/workflows/docs_release.yml index e0fdb0c2f7b..da67edd4aa1 100644 --- a/.github/workflows/docs_release.yml +++ b/.github/workflows/docs_release.yml @@ -116,8 +116,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" diff --git a/.github/workflows/jepsen.yml b/.github/workflows/jepsen.yml index 1682cd1e812..a8b04af5773 100644 --- a/.github/workflows/jepsen.yml +++ b/.github/workflows/jepsen.yml @@ -36,8 +36,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d3a303eb7ab..6f2fd5d678d 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -112,10 +112,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" CompatibilityCheck: needs: [BuilderDebRelease] @@ -146,10 +144,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" SharedBuildSmokeTest: needs: [BuilderDebShared] @@ -180,10 +176,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ######################################################################################### #################################### ORDINARY BUILDS #################################### @@ -230,10 +224,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderDebAarch64: needs: [DockerHubPush] @@ -273,10 +265,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinRelease: needs: [DockerHubPush] @@ -320,56 +310,9 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderBinGCC: - # needs: [DockerHubPush] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=binary_gcc - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v2 - # with: - # name: changed_images - # path: ${{ env.IMAGES_PATH }} - # - name: Clear repository - # run: | - # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - # - name: Check out repository code - # uses: actions/checkout@v2 - # - name: Build - # run: | - # git -C "$GITHUB_WORKSPACE" submodule sync --recursive - # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v2 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAsan: needs: [DockerHubPush] runs-on: [self-hosted, builder] @@ -410,10 +353,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebUBsan: needs: [DockerHubPush] @@ -455,10 +396,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebTsan: needs: [DockerHubPush] @@ -500,10 +439,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebMsan: needs: [DockerHubPush] @@ -545,10 +482,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebDebug: needs: [DockerHubPush] @@ -590,10 +525,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ########################################################################################## ##################################### SPECIAL BUILDS ##################################### @@ -638,10 +571,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinClangTidy: needs: [DockerHubPush] @@ -683,10 +614,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwin: needs: [DockerHubPush] @@ -730,10 +659,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinAarch64: needs: [DockerHubPush] @@ -777,10 +704,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinFreeBSD: needs: [DockerHubPush] @@ -824,10 +749,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwinAarch64: needs: [DockerHubPush] @@ -871,10 +794,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinPPC64: needs: [DockerHubPush] @@ -918,10 +839,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinAmd64SSE2: needs: [DockerHubPush] @@ -965,10 +884,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ############################################################################################ ##################################### Docker images ####################################### @@ -995,10 +912,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################ ##################################### BUILD REPORTER ####################################### @@ -1045,10 +960,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderSpecialReport: needs: @@ -1092,10 +1005,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ########################### FUNCTIONAl STATELESS TESTS ####################################### @@ -1132,10 +1043,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseDatabaseOrdinary: needs: [BuilderDebRelease] @@ -1169,10 +1078,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseDatabaseReplicated0: needs: [BuilderDebRelease] @@ -1208,10 +1115,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseDatabaseReplicated1: needs: [BuilderDebRelease] @@ -1247,10 +1152,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseS3: needs: [BuilderDebRelease] @@ -1284,10 +1187,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] @@ -1321,10 +1222,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan0: needs: [BuilderDebAsan] @@ -1360,10 +1259,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan1: needs: [BuilderDebAsan] @@ -1399,10 +1296,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan0: needs: [BuilderDebTsan] @@ -1438,10 +1333,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan1: needs: [BuilderDebTsan] @@ -1477,10 +1370,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan2: needs: [BuilderDebTsan] @@ -1516,10 +1407,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestUBsan: needs: [BuilderDebUBsan] @@ -1553,10 +1442,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan0: needs: [BuilderDebMsan] @@ -1592,10 +1479,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan1: needs: [BuilderDebMsan] @@ -1631,10 +1516,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan2: needs: [BuilderDebMsan] @@ -1670,10 +1553,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug0: needs: [BuilderDebDebug] @@ -1709,10 +1590,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug1: needs: [BuilderDebDebug] @@ -1748,10 +1627,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug2: needs: [BuilderDebDebug] @@ -1787,10 +1664,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### @@ -1827,10 +1702,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAarch64: needs: [BuilderDebAarch64] @@ -1864,10 +1737,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAsan: needs: [BuilderDebAsan] @@ -1901,10 +1772,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestTsan: needs: [BuilderDebTsan] @@ -1938,10 +1807,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestMsan: needs: [BuilderDebMsan] @@ -1975,10 +1842,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestUBsan: needs: [BuilderDebUBsan] @@ -2012,10 +1877,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestDebug: needs: [BuilderDebDebug] @@ -2049,10 +1912,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ######################################### STRESS TESTS ####################################### @@ -2088,10 +1949,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestTsan: needs: [BuilderDebTsan] @@ -2128,10 +1987,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestMsan: needs: [BuilderDebMsan] @@ -2164,10 +2021,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestUBsan: needs: [BuilderDebUBsan] @@ -2200,10 +2055,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestDebug: needs: [BuilderDebDebug] @@ -2236,10 +2089,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -2277,10 +2128,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan1: needs: [BuilderDebAsan] @@ -2315,10 +2164,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan2: needs: [BuilderDebAsan] @@ -2353,10 +2200,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan0: needs: [BuilderDebTsan] @@ -2391,10 +2236,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan1: needs: [BuilderDebTsan] @@ -2429,10 +2272,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan2: needs: [BuilderDebTsan] @@ -2467,10 +2308,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan3: needs: [BuilderDebTsan] @@ -2505,10 +2344,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease0: needs: [BuilderDebRelease] @@ -2543,10 +2380,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease1: needs: [BuilderDebRelease] @@ -2581,10 +2416,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ##################################### AST FUZZERS ############################################ @@ -2620,10 +2453,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestTsan: needs: [BuilderDebTsan] @@ -2656,10 +2487,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestUBSan: needs: [BuilderDebUBsan] @@ -2692,10 +2521,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestMSan: needs: [BuilderDebMsan] @@ -2728,10 +2555,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestDebug: needs: [BuilderDebDebug] @@ -2764,10 +2589,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### UNIT TESTS ############################################# @@ -2803,10 +2626,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsReleaseClang: needs: [BuilderBinRelease] @@ -2839,10 +2660,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" # UnitTestsReleaseGCC: # needs: [BuilderBinGCC] @@ -2875,10 +2694,8 @@ jobs: # - name: Cleanup # if: always() # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" UnitTestsTsan: needs: [BuilderDebTsan] @@ -2911,10 +2728,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsMsan: needs: [BuilderDebMsan] @@ -2947,10 +2762,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsUBsan: needs: [BuilderDebUBsan] @@ -2983,10 +2796,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### PERFORMANCE TESTS ###################################### @@ -3024,10 +2835,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-1: needs: [BuilderDebRelease] @@ -3062,10 +2871,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-2: needs: [BuilderDebRelease] @@ -3100,10 +2907,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-3: needs: [BuilderDebRelease] @@ -3138,10 +2943,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FinishCheck: needs: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e712ada1551..801f7eda94a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -119,8 +119,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 3f4e5d7bb00..24a1c6bb714 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -141,10 +141,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FastTest: needs: DockerHubPush @@ -177,10 +175,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" CompatibilityCheck: needs: [BuilderDebRelease] @@ -211,10 +207,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" SharedBuildSmokeTest: needs: [BuilderDebShared] @@ -245,10 +239,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ######################################################################################### #################################### ORDINARY BUILDS #################################### @@ -295,10 +287,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderBinRelease: needs: [DockerHubPush, FastTest, StyleCheck] @@ -340,10 +330,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush, FastTest, StyleCheck] @@ -387,10 +375,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAsan: needs: [DockerHubPush, FastTest, StyleCheck] @@ -432,10 +418,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebUBsan: needs: [DockerHubPush, FastTest, StyleCheck] @@ -477,10 +461,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebTsan: needs: [DockerHubPush, FastTest, StyleCheck] @@ -522,10 +504,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebMsan: needs: [DockerHubPush, FastTest, StyleCheck] @@ -567,10 +547,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebDebug: needs: [DockerHubPush, FastTest, StyleCheck] @@ -612,10 +590,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ########################################################################################## ##################################### SPECIAL BUILDS ##################################### @@ -660,10 +636,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinClangTidy: needs: [DockerHubPush, FastTest, StyleCheck] @@ -705,10 +679,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwin: needs: [DockerHubPush, FastTest, StyleCheck] @@ -750,10 +722,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinAarch64: needs: [DockerHubPush, FastTest, StyleCheck] @@ -795,10 +765,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinFreeBSD: needs: [DockerHubPush, FastTest, StyleCheck] @@ -840,10 +808,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwinAarch64: needs: [DockerHubPush, FastTest, StyleCheck] @@ -885,10 +851,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinPPC64: needs: [DockerHubPush, FastTest, StyleCheck] @@ -930,10 +894,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinAmd64SSE2: needs: [DockerHubPush, FastTest, StyleCheck] @@ -975,10 +937,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ############################################################################################ ##################################### Docker images ####################################### @@ -1005,10 +965,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################ ##################################### BUILD REPORTER ####################################### @@ -1055,10 +1013,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderSpecialReport: needs: @@ -1103,10 +1059,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ########################### FUNCTIONAl STATELESS TESTS ####################################### @@ -1143,10 +1097,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseDatabaseReplicated0: needs: [BuilderDebRelease] @@ -1182,10 +1134,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseDatabaseReplicated1: needs: [BuilderDebRelease] @@ -1221,10 +1171,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseWideParts: needs: [BuilderDebRelease] @@ -1258,10 +1206,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseS3: needs: [BuilderDebRelease] @@ -1295,10 +1241,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Debug0: needs: [BuilderDebDebug] @@ -1334,8 +1278,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Debug1: needs: [BuilderDebDebug] @@ -1371,8 +1315,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Debug2: needs: [BuilderDebDebug] @@ -1408,8 +1352,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Tsan0: needs: [BuilderDebTsan] @@ -1445,8 +1389,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Tsan1: needs: [BuilderDebTsan] @@ -1482,8 +1426,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestS3Tsan2: needs: [BuilderDebTsan] @@ -1519,8 +1463,8 @@ jobs: - name: Cleanup if: always() run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] @@ -1554,10 +1498,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan0: needs: [BuilderDebAsan] @@ -1593,10 +1535,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan1: needs: [BuilderDebAsan] @@ -1632,10 +1572,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan0: needs: [BuilderDebTsan] @@ -1671,10 +1609,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan1: needs: [BuilderDebTsan] @@ -1710,10 +1646,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan2: needs: [BuilderDebTsan] @@ -1749,10 +1683,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestUBsan: needs: [BuilderDebUBsan] @@ -1786,10 +1718,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan0: needs: [BuilderDebMsan] @@ -1825,10 +1755,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan1: needs: [BuilderDebMsan] @@ -1864,10 +1792,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan2: needs: [BuilderDebMsan] @@ -1903,10 +1829,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug0: needs: [BuilderDebDebug] @@ -1942,10 +1866,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug1: needs: [BuilderDebDebug] @@ -1981,10 +1903,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug2: needs: [BuilderDebDebug] @@ -2020,10 +1940,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestFlakyCheck: needs: [BuilderDebAsan] @@ -2057,10 +1975,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" TestsBugfixCheck: runs-on: [self-hosted, stress-tester] @@ -2104,10 +2020,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### @@ -2144,10 +2058,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAarch64: needs: [BuilderDebAarch64] @@ -2181,10 +2093,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAsan: needs: [BuilderDebAsan] @@ -2218,10 +2128,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestTsan: needs: [BuilderDebTsan] @@ -2255,10 +2163,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestMsan: needs: [BuilderDebMsan] @@ -2292,10 +2198,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestUBsan: needs: [BuilderDebUBsan] @@ -2329,10 +2233,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestDebug: needs: [BuilderDebDebug] @@ -2366,10 +2268,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ######################################### STRESS TESTS ####################################### @@ -2405,10 +2305,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestTsan: needs: [BuilderDebTsan] @@ -2445,10 +2343,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestMsan: needs: [BuilderDebMsan] @@ -2481,10 +2377,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestUBsan: needs: [BuilderDebUBsan] @@ -2517,10 +2411,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestDebug: needs: [BuilderDebDebug] @@ -2553,10 +2445,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ##################################### AST FUZZERS ############################################ @@ -2592,10 +2482,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestTsan: needs: [BuilderDebTsan] @@ -2628,10 +2516,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestUBSan: needs: [BuilderDebUBsan] @@ -2664,10 +2550,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestMSan: needs: [BuilderDebMsan] @@ -2700,10 +2584,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ASTFuzzerTestDebug: needs: [BuilderDebDebug] @@ -2736,10 +2618,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -2777,10 +2657,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan1: needs: [BuilderDebAsan] @@ -2815,10 +2693,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan2: needs: [BuilderDebAsan] @@ -2853,10 +2729,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan0: needs: [BuilderDebTsan] @@ -2891,10 +2765,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan1: needs: [BuilderDebTsan] @@ -2929,10 +2801,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan2: needs: [BuilderDebTsan] @@ -2967,10 +2837,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan3: needs: [BuilderDebTsan] @@ -3005,10 +2873,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease0: needs: [BuilderDebRelease] @@ -3043,10 +2909,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease1: needs: [BuilderDebRelease] @@ -3081,10 +2945,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsFlakyCheck: needs: [BuilderDebAsan] @@ -3117,10 +2979,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### UNIT TESTS ############################################# @@ -3156,10 +3016,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsReleaseClang: needs: [BuilderBinRelease] @@ -3192,10 +3050,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsTsan: needs: [BuilderDebTsan] @@ -3228,10 +3084,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsMsan: needs: [BuilderDebMsan] @@ -3264,10 +3118,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" UnitTestsUBsan: needs: [BuilderDebUBsan] @@ -3300,10 +3152,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### PERFORMANCE TESTS ###################################### @@ -3341,10 +3191,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-1: needs: [BuilderDebRelease] @@ -3379,10 +3227,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-2: needs: [BuilderDebRelease] @@ -3417,10 +3263,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonX86-3: needs: [BuilderDebRelease] @@ -3455,10 +3299,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonAarch-0: needs: [BuilderDebAarch64] @@ -3493,10 +3335,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonAarch-1: needs: [BuilderDebAarch64] @@ -3531,10 +3371,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonAarch-2: needs: [BuilderDebAarch64] @@ -3569,10 +3407,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" PerformanceComparisonAarch-3: needs: [BuilderDebAarch64] @@ -3607,10 +3443,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# ###################################### JEPSEN TESTS ######################################### diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ae905aa62ba..0b0f125d641 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,10 +30,11 @@ jobs: cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cd "$REPO_COPY" # Download and push packages to artifactory - python3 ./tests/ci/push_to_artifactory.py --release "${{ github.ref }}" \ - --commit '${{ github.sha }}' --artifactory-url "${{ secrets.JFROG_ARTIFACTORY_URL }}" --all + python3 ./tests/ci/push_to_artifactory.py --release '${{ github.ref }}' \ + --commit '${{ github.sha }}' --artifactory-url '${{ secrets.JFROG_ARTIFACTORY_URL }}' --all # Download macos binaries to ${{runner.temp}}/download_binary - python3 ./tests/ci/download_binary.py binary_darwin binary_darwin_aarch64 + python3 ./tests/ci/download_binary.py --version '${{ github.ref }}' \ + --commit '${{ github.sha }}' binary_darwin binary_darwin_aarch64 mv '${{runner.temp}}/download_binary/'clickhouse-* '${{runner.temp}}/push_to_artifactory' - name: Upload packages to release assets uses: svenstaro/upload-release-action@v2 @@ -65,8 +66,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index f579d1fee63..8f42ca92646 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -103,10 +103,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ######################################################################################### #################################### ORDINARY BUILDS #################################### @@ -153,10 +151,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush] @@ -196,10 +192,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAsan: needs: [DockerHubPush] @@ -241,10 +235,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebUBsan: needs: [DockerHubPush] @@ -286,10 +278,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebTsan: needs: [DockerHubPush] @@ -331,10 +321,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebMsan: needs: [DockerHubPush] @@ -376,10 +364,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebDebug: needs: [DockerHubPush] @@ -421,10 +407,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwin: needs: [DockerHubPush] @@ -468,10 +452,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwinAarch64: needs: [DockerHubPush] @@ -515,10 +497,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ############################################################################################ ##################################### Docker images ####################################### @@ -545,10 +525,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################ ##################################### BUILD REPORTER ####################################### @@ -594,10 +572,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderSpecialReport: needs: @@ -634,10 +610,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ########################### FUNCTIONAl STATELESS TESTS ####################################### @@ -674,10 +648,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] @@ -711,10 +683,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan0: needs: [BuilderDebAsan] @@ -750,10 +720,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAsan1: needs: [BuilderDebAsan] @@ -789,10 +757,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan0: needs: [BuilderDebTsan] @@ -828,10 +794,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan1: needs: [BuilderDebTsan] @@ -867,10 +831,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestTsan2: needs: [BuilderDebTsan] @@ -906,10 +868,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestUBsan: needs: [BuilderDebUBsan] @@ -943,10 +903,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan0: needs: [BuilderDebMsan] @@ -982,10 +940,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan1: needs: [BuilderDebMsan] @@ -1021,10 +977,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestMsan2: needs: [BuilderDebMsan] @@ -1060,10 +1014,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug0: needs: [BuilderDebDebug] @@ -1099,10 +1051,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug1: needs: [BuilderDebDebug] @@ -1138,10 +1088,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestDebug2: needs: [BuilderDebDebug] @@ -1177,10 +1125,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### @@ -1217,10 +1163,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAarch64: needs: [BuilderDebAarch64] @@ -1254,10 +1198,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestAsan: needs: [BuilderDebAsan] @@ -1291,10 +1233,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestTsan: needs: [BuilderDebTsan] @@ -1328,10 +1268,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestMsan: needs: [BuilderDebMsan] @@ -1365,10 +1303,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestUBsan: needs: [BuilderDebUBsan] @@ -1402,10 +1338,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FunctionalStatefulTestDebug: needs: [BuilderDebDebug] @@ -1439,10 +1373,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################## ######################################### STRESS TESTS ####################################### @@ -1478,10 +1410,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestTsan: needs: [BuilderDebTsan] @@ -1518,10 +1448,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestMsan: needs: [BuilderDebMsan] @@ -1554,10 +1482,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestUBsan: needs: [BuilderDebUBsan] @@ -1590,10 +1516,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" StressTestDebug: needs: [BuilderDebDebug] @@ -1626,10 +1550,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -1667,10 +1589,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan1: needs: [BuilderDebAsan] @@ -1705,10 +1625,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsAsan2: needs: [BuilderDebAsan] @@ -1743,10 +1661,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan0: needs: [BuilderDebTsan] @@ -1781,10 +1697,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan1: needs: [BuilderDebTsan] @@ -1819,10 +1733,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan2: needs: [BuilderDebTsan] @@ -1857,10 +1769,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsTsan3: needs: [BuilderDebTsan] @@ -1895,10 +1805,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease0: needs: [BuilderDebRelease] @@ -1933,10 +1841,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" IntegrationTestsRelease1: needs: [BuilderDebRelease] @@ -1971,10 +1877,8 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" FinishCheck: needs: diff --git a/.github/workflows/tags_stable.yml b/.github/workflows/tags_stable.yml index 9711f7688cb..a9172a8a2e2 100644 --- a/.github/workflows/tags_stable.yml +++ b/.github/workflows/tags_stable.yml @@ -43,6 +43,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }} run: | ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv + ./utils/list-versions/update-docker-version.sh GID=$(id -g "${UID}") docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 \ --volume="${GITHUB_WORKSPACE}:/ClickHouse" clickhouse/style-test \ diff --git a/.github/workflows/woboq.yml b/.github/workflows/woboq.yml index dd29131e9dc..b928a4a8d3d 100644 --- a/.github/workflows/woboq.yml +++ b/.github/workflows/woboq.yml @@ -37,8 +37,6 @@ jobs: - name: Cleanup if: always() run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" diff --git a/.gitignore b/.gitignore index e517dfd63c2..dd632eba85d 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,10 @@ cmake_install.cmake CTestTestfile.cmake *.a *.o +*.so +*.dll +*.lib +*.dylib cmake-build-* # Python cache diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index b7c18110503..75c48f690f8 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -220,6 +220,35 @@ ReplxxLineReader::ReplxxLineReader( rx.bind_key(Replxx::KEY::control('W'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::KILL_TO_WHITESPACE_ON_LEFT, code); }); rx.bind_key(Replxx::KEY::meta('E'), [this](char32_t) { openEditor(); return Replxx::ACTION_RESULT::CONTINUE; }); + + /// readline insert-comment + auto insert_comment_action = [this](char32_t code) + { + replxx::Replxx::State state(rx.get_state()); + const char * line = state.text(); + const char * line_end = line + strlen(line); + + std::string commented_line; + if (std::find(line, line_end, '\n') != line_end) + { + /// If query has multiple lines, multiline comment is used over + /// commenting each line separately for easier uncomment (though + /// with invoking editor it is simpler to uncomment multiple lines) + /// + /// Note, that using multiline comment is OK even with nested + /// comments, since nested comments are supported. + commented_line = fmt::format("/* {} */", state.text()); + } + else + { + // In a simplest case use simple comment. + commented_line = fmt::format("-- {}", state.text()); + } + rx.set_state(replxx::Replxx::State(commented_line.c_str(), commented_line.size())); + + return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); + }; + rx.bind_key(Replxx::KEY::meta('#'), insert_comment_action); } ReplxxLineReader::~ReplxxLineReader() diff --git a/base/base/bit_cast.h b/base/base/bit_cast.h index d1246b45590..b2b6915764d 100644 --- a/base/base/bit_cast.h +++ b/base/base/bit_cast.h @@ -5,8 +5,9 @@ #include -/** \brief Returns value `from` converted to type `To` while retaining bit representation. - * `To` and `From` must satisfy `CopyConstructible`. +/** Returns value `from` converted to type `To` while retaining bit representation. + * `To` and `From` must satisfy `CopyConstructible`. + * In contrast to std::bit_cast can cast types of different width. */ template std::decay_t bit_cast(const From & from) @@ -15,13 +16,3 @@ std::decay_t bit_cast(const From & from) memcpy(static_cast(&res), &from, std::min(sizeof(res), sizeof(from))); return res; } - -/** \brief Returns value `from` converted to type `To` while retaining bit representation. - * `To` and `From` must satisfy `CopyConstructible`. - */ -template -std::decay_t safe_bit_cast(const From & from) -{ - static_assert(sizeof(To) == sizeof(From), "bit cast on types of different width"); - return bit_cast(from); -} diff --git a/base/base/phdr_cache.h b/base/base/phdr_cache.h index b522710c4c4..a6e129db34d 100644 --- a/base/base/phdr_cache.h +++ b/base/base/phdr_cache.h @@ -8,6 +8,7 @@ * As a drawback, this only works if no dynamic object unloading happens after this point. * This function is thread-safe. You should call it to update cache after loading new shared libraries. * Otherwise exception handling from dlopened libraries won't work (will call std::terminate immediately). + * NOTE: dlopen is forbidden in our code. * * NOTE: It is disabled with Thread Sanitizer because TSan can only use original "dl_iterate_phdr" function. */ diff --git a/cmake/clang_tidy.cmake b/cmake/clang_tidy.cmake index fc25c68b11a..200282234ca 100644 --- a/cmake/clang_tidy.cmake +++ b/cmake/clang_tidy.cmake @@ -3,7 +3,7 @@ option (ENABLE_CLANG_TIDY "Use clang-tidy static analyzer" OFF) if (ENABLE_CLANG_TIDY) - find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12") + find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12") if (CLANG_TIDY_PATH) message(STATUS diff --git a/cmake/target.cmake b/cmake/target.cmake index 0fb5e8a20de..ae360758701 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -45,6 +45,7 @@ if (CMAKE_CROSSCOMPILING) endif () if (USE_MUSL) + # use of undeclared identifier 'PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP' set (ENABLE_SENTRY OFF CACHE INTERNAL "") set (ENABLE_ODBC OFF CACHE INTERNAL "") set (ENABLE_GRPC OFF CACHE INTERNAL "") diff --git a/contrib/c-ares-cmake/CMakeLists.txt b/contrib/c-ares-cmake/CMakeLists.txt index 603c1f8b65c..4b1170f9dd1 100644 --- a/contrib/c-ares-cmake/CMakeLists.txt +++ b/contrib/c-ares-cmake/CMakeLists.txt @@ -1,35 +1,95 @@ -# Choose to build static or shared library for c-ares. -if (USE_STATIC_LIBRARIES) - set(CARES_STATIC ON CACHE BOOL "" FORCE) - set(CARES_SHARED OFF CACHE BOOL "" FORCE) -else () - set(CARES_STATIC OFF CACHE BOOL "" FORCE) - set(CARES_SHARED ON CACHE BOOL "" FORCE) -endif () +set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/c-ares") -# Disable looking for libnsl on a platforms that has gethostbyname in glibc -# -# c-ares searching for gethostbyname in the libnsl library, however in the -# version that shipped with gRPC it doing it wrong [1], since it uses -# CHECK_LIBRARY_EXISTS(), which will return TRUE even if the function exists in -# another dependent library. The upstream already contains correct macro [2], -# but it is not included in gRPC (even upstream gRPC, not the one that is -# shipped with clickhousee). -# -# [1]: https://github.com/c-ares/c-ares/blob/e982924acee7f7313b4baa4ee5ec000c5e373c30/CMakeLists.txt#L125 -# [2]: https://github.com/c-ares/c-ares/blob/44fbc813685a1fa8aa3f27fcd7544faf612d376a/CMakeLists.txt#L146 -# -# And because if you by some reason have libnsl [3] installed, clickhouse will -# reject to start w/o it. While this is completelly different library. -# -# [3]: https://packages.debian.org/bullseye/libnsl2 -if (NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS") - set(HAVE_LIBNSL OFF CACHE BOOL "" FORCE) +# Generated from contrib/c-ares/src/lib/Makefile.inc +SET(SRCS + "${LIBRARY_DIR}/src/lib/ares__addrinfo2hostent.c" + "${LIBRARY_DIR}/src/lib/ares__addrinfo_localhost.c" + "${LIBRARY_DIR}/src/lib/ares__close_sockets.c" + "${LIBRARY_DIR}/src/lib/ares__get_hostent.c" + "${LIBRARY_DIR}/src/lib/ares__parse_into_addrinfo.c" + "${LIBRARY_DIR}/src/lib/ares__readaddrinfo.c" + "${LIBRARY_DIR}/src/lib/ares__sortaddrinfo.c" + "${LIBRARY_DIR}/src/lib/ares__read_line.c" + "${LIBRARY_DIR}/src/lib/ares__timeval.c" + "${LIBRARY_DIR}/src/lib/ares_android.c" + "${LIBRARY_DIR}/src/lib/ares_cancel.c" + "${LIBRARY_DIR}/src/lib/ares_data.c" + "${LIBRARY_DIR}/src/lib/ares_destroy.c" + "${LIBRARY_DIR}/src/lib/ares_expand_name.c" + "${LIBRARY_DIR}/src/lib/ares_expand_string.c" + "${LIBRARY_DIR}/src/lib/ares_fds.c" + "${LIBRARY_DIR}/src/lib/ares_free_hostent.c" + "${LIBRARY_DIR}/src/lib/ares_free_string.c" + "${LIBRARY_DIR}/src/lib/ares_freeaddrinfo.c" + "${LIBRARY_DIR}/src/lib/ares_getaddrinfo.c" + "${LIBRARY_DIR}/src/lib/ares_getenv.c" + "${LIBRARY_DIR}/src/lib/ares_gethostbyaddr.c" + "${LIBRARY_DIR}/src/lib/ares_gethostbyname.c" + "${LIBRARY_DIR}/src/lib/ares_getnameinfo.c" + "${LIBRARY_DIR}/src/lib/ares_getsock.c" + "${LIBRARY_DIR}/src/lib/ares_init.c" + "${LIBRARY_DIR}/src/lib/ares_library_init.c" + "${LIBRARY_DIR}/src/lib/ares_llist.c" + "${LIBRARY_DIR}/src/lib/ares_mkquery.c" + "${LIBRARY_DIR}/src/lib/ares_create_query.c" + "${LIBRARY_DIR}/src/lib/ares_nowarn.c" + "${LIBRARY_DIR}/src/lib/ares_options.c" + "${LIBRARY_DIR}/src/lib/ares_parse_a_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_aaaa_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_caa_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_mx_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_naptr_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_ns_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_ptr_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_soa_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_srv_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_txt_reply.c" + "${LIBRARY_DIR}/src/lib/ares_parse_uri_reply.c" + "${LIBRARY_DIR}/src/lib/ares_platform.c" + "${LIBRARY_DIR}/src/lib/ares_process.c" + "${LIBRARY_DIR}/src/lib/ares_query.c" + "${LIBRARY_DIR}/src/lib/ares_search.c" + "${LIBRARY_DIR}/src/lib/ares_send.c" + "${LIBRARY_DIR}/src/lib/ares_strcasecmp.c" + "${LIBRARY_DIR}/src/lib/ares_strdup.c" + "${LIBRARY_DIR}/src/lib/ares_strerror.c" + "${LIBRARY_DIR}/src/lib/ares_strsplit.c" + "${LIBRARY_DIR}/src/lib/ares_timeout.c" + "${LIBRARY_DIR}/src/lib/ares_version.c" + "${LIBRARY_DIR}/src/lib/ares_writev.c" + "${LIBRARY_DIR}/src/lib/bitncmp.c" + "${LIBRARY_DIR}/src/lib/inet_net_pton.c" + "${LIBRARY_DIR}/src/lib/inet_ntop.c" + "${LIBRARY_DIR}/src/lib/windows_port.c" +) + +if (USE_STATIC_LIBRARIES) + add_library(_c-ares STATIC ${SRCS}) + target_compile_definitions(_c-ares PUBLIC CARES_STATICLIB) +else() + add_library(_c-ares SHARED ${SRCS}) + target_compile_definitions(_c-ares PUBLIC CARES_BUILDING_LIBRARY) endif() -# Force use of c-ares inet_net_pton instead of libresolv one -set(HAVE_INET_NET_PTON OFF CACHE BOOL "" FORCE) +target_compile_definitions(_c-ares PRIVATE HAVE_CONFIG_H=1) -add_subdirectory("../c-ares/" "../c-ares/") +target_include_directories(_c-ares SYSTEM PUBLIC + "${LIBRARY_DIR}/src/lib" + "${LIBRARY_DIR}/include" +) -add_library(ch_contrib::c-ares ALIAS c-ares) \ No newline at end of file +# Platform-specific include directories. The original build system does a lot of checks to eventually generate two header files with defines: +# ares_build.h and ares_config.h. To update, run the original CMake build in c-ares for each platform and copy the headers into the +# platform-specific folder. +# For the platform-specific compile definitions, see c-ares top-level CMakeLists.txt. +if (OS_LINUX) + target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/linux") + target_compile_definitions(_c-ares PRIVATE -D_GNU_SOURCE -D_POSIX_C_SOURCE=199309L -D_XOPEN_SOURCE=600) +elseif (OS_DARWIN) + target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/darwin") + target_compile_definitions(_c-ares PRIVATE -D_DARWIN_C_SOURCE) +elseif (OS_FREEBSD) + target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/freebsd") +endif() + +add_library(ch_contrib::c-ares ALIAS _c-ares) diff --git a/contrib/c-ares-cmake/darwin/ares_build.h b/contrib/c-ares-cmake/darwin/ares_build.h new file mode 100644 index 00000000000..bf7402e7997 --- /dev/null +++ b/contrib/c-ares-cmake/darwin/ares_build.h @@ -0,0 +1,43 @@ +#ifndef __CARES_BUILD_H +#define __CARES_BUILD_H + +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Prefix names with CARES_ to make sure they don't conflict with other config.h + * files. We need to include some dependent headers that may be system specific + * for C-Ares */ +#define CARES_HAVE_SYS_TYPES_H +#define CARES_HAVE_SYS_SOCKET_H +/* #undef CARES_HAVE_WINDOWS_H */ +/* #undef CARES_HAVE_WS2TCPIP_H */ +/* #undef CARES_HAVE_WINSOCK2_H */ +/* #undef CARES_HAVE_WINDOWS_H */ +#define CARES_HAVE_ARPA_NAMESER_H +#define CARES_HAVE_ARPA_NAMESER_COMPAT_H + +#ifdef CARES_HAVE_SYS_TYPES_H +# include +#endif + +#ifdef CARES_HAVE_SYS_SOCKET_H +# include +#endif + +#ifdef CARES_HAVE_WINSOCK2_H +# include +#endif + +#ifdef CARES_HAVE_WS2TCPIP_H +# include +#endif + +#ifdef CARES_HAVE_WINDOWS_H +# include +#endif + + +typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t; +typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t; + +#endif /* __CARES_BUILD_H */ diff --git a/contrib/c-ares-cmake/darwin/ares_config.h b/contrib/c-ares-cmake/darwin/ares_config.h new file mode 100644 index 00000000000..64af3836f3f --- /dev/null +++ b/contrib/c-ares-cmake/darwin/ares_config.h @@ -0,0 +1,432 @@ +/* Generated from ares_config.h.cmake */ + +/* Define if building universal (internal helper macro) */ +#undef AC_APPLE_UNIVERSAL_BUILD + +/* define this if ares is built for a big endian system */ +#undef ARES_BIG_ENDIAN + +/* when building as static part of libcurl */ +#undef BUILDING_LIBCURL + +/* Defined for build that exposes internal static functions for testing. */ +#undef CARES_EXPOSE_STATICS + +/* Defined for build with symbol hiding. */ +#undef CARES_SYMBOL_HIDING + +/* Definition to make a library symbol externally visible. */ +#undef CARES_SYMBOL_SCOPE_EXTERN + +/* Use resolver library to configure cares */ +/* #undef CARES_USE_LIBRESOLV */ + +/* if a /etc/inet dir is being used */ +#undef ETC_INET + +/* Define to the type of arg 2 for gethostname. */ +#define GETHOSTNAME_TYPE_ARG2 size_t + +/* Define to the type qualifier of arg 1 for getnameinfo. */ +#define GETNAMEINFO_QUAL_ARG1 + +/* Define to the type of arg 1 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG1 struct sockaddr * + +/* Define to the type of arg 2 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG2 socklen_t + +/* Define to the type of args 4 and 6 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG46 socklen_t + +/* Define to the type of arg 7 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG7 int + +/* Specifies the number of arguments to getservbyport_r */ +#define GETSERVBYPORT_R_ARGS + +/* Specifies the number of arguments to getservbyname_r */ +#define GETSERVBYNAME_R_ARGS + +/* Define to 1 if you have AF_INET6. */ +#define HAVE_AF_INET6 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_COMPAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H + +/* Define to 1 if you have the `bitncmp' function. */ +/* #undef HAVE_BITNCMP */ + +/* Define to 1 if bool is an available type. */ +#define HAVE_BOOL_T + +/* Define to 1 if you have the clock_gettime function and monotonic timer. */ +#define HAVE_CLOCK_GETTIME_MONOTONIC + +/* Define to 1 if you have the closesocket function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the CloseSocket camel case function. */ +/* #undef HAVE_CLOSESOCKET_CAMEL */ + +/* Define to 1 if you have the connect function. */ +#define HAVE_CONNECT + +/* define if the compiler supports basic C++11 syntax */ +/* #undef HAVE_CXX11 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H + +/* Define to 1 if you have the fcntl function. */ +#define HAVE_FCNTL + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H + +/* Define to 1 if you have a working fcntl O_NONBLOCK function. */ +#define HAVE_FCNTL_O_NONBLOCK + +/* Define to 1 if you have the freeaddrinfo function. */ +#define HAVE_FREEADDRINFO + +/* Define to 1 if you have a working getaddrinfo function. */ +#define HAVE_GETADDRINFO + +/* Define to 1 if the getaddrinfo function is threadsafe. */ +#define HAVE_GETADDRINFO_THREADSAFE + +/* Define to 1 if you have the getenv function. */ +#define HAVE_GETENV + +/* Define to 1 if you have the gethostbyaddr function. */ +#define HAVE_GETHOSTBYADDR + +/* Define to 1 if you have the gethostbyname function. */ +#define HAVE_GETHOSTBYNAME + +/* Define to 1 if you have the gethostname function. */ +#define HAVE_GETHOSTNAME + +/* Define to 1 if you have the getnameinfo function. */ +#define HAVE_GETNAMEINFO + +/* Define to 1 if you have the getservbyport_r function. */ +/* #undef HAVE_GETSERVBYPORT_R */ + +/* Define to 1 if you have the getservbyname_r function. */ +/* #undef HAVE_GETSERVBYNAME_R */ + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY + +/* Define to 1 if you have the `if_indextoname' function. */ +#define HAVE_IF_INDEXTONAME + +/* Define to 1 if you have a IPv6 capable working inet_net_pton function. */ +/* #undef HAVE_INET_NET_PTON */ + +/* Define to 1 if you have a IPv6 capable working inet_ntop function. */ +#define HAVE_INET_NTOP + +/* Define to 1 if you have a IPv6 capable working inet_pton function. */ +#define HAVE_INET_PTON + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H + +/* Define to 1 if you have the ioctl function. */ +#define HAVE_IOCTL + +/* Define to 1 if you have the ioctlsocket function. */ +/* #undef HAVE_IOCTLSOCKET */ + +/* Define to 1 if you have the IoctlSocket camel case function. */ +/* #undef HAVE_IOCTLSOCKET_CAMEL */ + +/* Define to 1 if you have a working IoctlSocket camel case FIONBIO function. + */ +/* #undef HAVE_IOCTLSOCKET_CAMEL_FIONBIO */ + +/* Define to 1 if you have a working ioctlsocket FIONBIO function. */ +/* #undef HAVE_IOCTLSOCKET_FIONBIO */ + +/* Define to 1 if you have a working ioctl FIONBIO function. */ +#define HAVE_IOCTL_FIONBIO + +/* Define to 1 if you have a working ioctl SIOCGIFADDR function. */ +#define HAVE_IOCTL_SIOCGIFADDR + +/* Define to 1 if you have the `resolve' library (-lresolve). */ +/* #undef HAVE_LIBRESOLV */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H + +/* if your compiler supports LL */ +#define HAVE_LL + +/* Define to 1 if the compiler supports the 'long long' data type. */ +#define HAVE_LONGLONG + +/* Define to 1 if you have the malloc.h header file. */ +/* #undef HAVE_MALLOC_H */ + +/* Define to 1 if you have the memory.h header file. */ +#define HAVE_MEMORY_H + +/* Define to 1 if you have the MSG_NOSIGNAL flag. */ +/* #undef HAVE_MSG_NOSIGNAL */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H + +/* Define to 1 if you have PF_INET6. */ +#define HAVE_PF_INET6 + +/* Define to 1 if you have the recv function. */ +#define HAVE_RECV + +/* Define to 1 if you have the recvfrom function. */ +#define HAVE_RECVFROM + +/* Define to 1 if you have the send function. */ +#define HAVE_SEND + +/* Define to 1 if you have the setsockopt function. */ +#define HAVE_SETSOCKOPT + +/* Define to 1 if you have a working setsockopt SO_NONBLOCK function. */ +/* #undef HAVE_SETSOCKOPT_SO_NONBLOCK */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SIGNAL_H + +/* Define to 1 if sig_atomic_t is an available typedef. */ +#define HAVE_SIG_ATOMIC_T + +/* Define to 1 if sig_atomic_t is already defined as volatile. */ +/* #undef HAVE_SIG_ATOMIC_T_VOLATILE */ + +/* Define to 1 if your struct sockaddr_in6 has sin6_scope_id. */ +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID + +/* Define to 1 if you have the socket function. */ +#define HAVE_SOCKET + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SOCKET_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDBOOL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H + +/* Define to 1 if you have the strcasecmp function. */ +#define HAVE_STRCASECMP + +/* Define to 1 if you have the strcmpi function. */ +/* #undef HAVE_STRCMPI */ + +/* Define to 1 if you have the strdup function. */ +#define HAVE_STRDUP + +/* Define to 1 if you have the stricmp function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H + +/* Define to 1 if you have the strncasecmp function. */ +#define HAVE_STRNCASECMP + +/* Define to 1 if you have the strncmpi function. */ +/* #undef HAVE_STRNCMPI */ + +/* Define to 1 if you have the strnicmp function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_STROPTS_H */ + +/* Define to 1 if you have struct addrinfo. */ +#define HAVE_STRUCT_ADDRINFO + +/* Define to 1 if you have struct in6_addr. */ +#define HAVE_STRUCT_IN6_ADDR + +/* Define to 1 if you have struct sockaddr_in6. */ +#define HAVE_STRUCT_SOCKADDR_IN6 + +/* if struct sockaddr_storage is defined */ +#define HAVE_STRUCT_SOCKADDR_STORAGE + +/* Define to 1 if you have the timeval struct. */ +#define HAVE_STRUCT_TIMEVAL + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H + +/* Define to 1 if you have the windows.h header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the winsock2.h header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the winsock.h header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the writev function. */ +#define HAVE_WRITEV + +/* Define to 1 if you have the ws2tcpip.h header file. */ +/* #undef HAVE_WS2TCPIP_H */ + +/* Define to 1 if you have the __system_property_get function */ +#define HAVE___SYSTEM_PROPERTY_GET + +/* Define to 1 if you need the malloc.h header file even with stdlib.h */ +/* #undef NEED_MALLOC_H */ + +/* Define to 1 if you need the memory.h header file even with stdlib.h */ +/* #undef NEED_MEMORY_H */ + +/* a suitable file/device to read random data from */ +#define CARES_RANDOM_FILE "/dev/urandom" + +/* Define to the type qualifier pointed by arg 5 for recvfrom. */ +#define RECVFROM_QUAL_ARG5 + +/* Define to the type of arg 1 for recvfrom. */ +#define RECVFROM_TYPE_ARG1 int + +/* Define to the type pointed by arg 2 for recvfrom. */ +#define RECVFROM_TYPE_ARG2 void * + +/* Define to 1 if the type pointed by arg 2 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG2_IS_VOID 0 + +/* Define to the type of arg 3 for recvfrom. */ +#define RECVFROM_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recvfrom. */ +#define RECVFROM_TYPE_ARG4 int + +/* Define to the type pointed by arg 5 for recvfrom. */ +#define RECVFROM_TYPE_ARG5 struct sockaddr * + +/* Define to 1 if the type pointed by arg 5 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG5_IS_VOID 0 + +/* Define to the type pointed by arg 6 for recvfrom. */ +#define RECVFROM_TYPE_ARG6 socklen_t * + +/* Define to 1 if the type pointed by arg 6 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG6_IS_VOID 0 + +/* Define to the function return type for recvfrom. */ +#define RECVFROM_TYPE_RETV ssize_t + +/* Define to the type of arg 1 for recv. */ +#define RECV_TYPE_ARG1 int + +/* Define to the type of arg 2 for recv. */ +#define RECV_TYPE_ARG2 void * + +/* Define to the type of arg 3 for recv. */ +#define RECV_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recv. */ +#define RECV_TYPE_ARG4 int + +/* Define to the function return type for recv. */ +#define RECV_TYPE_RETV ssize_t + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE + +/* Define to the type qualifier of arg 2 for send. */ +#define SEND_QUAL_ARG2 + +/* Define to the type of arg 1 for send. */ +#define SEND_TYPE_ARG1 int + +/* Define to the type of arg 2 for send. */ +#define SEND_TYPE_ARG2 void * + +/* Define to the type of arg 3 for send. */ +#define SEND_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for send. */ +#define SEND_TYPE_ARG4 int + +/* Define to the function return type for send. */ +#define SEND_TYPE_RETV ssize_t + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME + +/* Define to disable non-blocking sockets. */ +#undef USE_BLOCKING_SOCKETS + +/* Define to avoid automatic inclusion of winsock.h */ +#undef WIN32_LEAN_AND_MEAN + +/* Type to use in place of in_addr_t when system does not provide it. */ +#undef in_addr_t + diff --git a/contrib/c-ares-cmake/freebsd/ares_build.h b/contrib/c-ares-cmake/freebsd/ares_build.h new file mode 100644 index 00000000000..bf7402e7997 --- /dev/null +++ b/contrib/c-ares-cmake/freebsd/ares_build.h @@ -0,0 +1,43 @@ +#ifndef __CARES_BUILD_H +#define __CARES_BUILD_H + +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Prefix names with CARES_ to make sure they don't conflict with other config.h + * files. We need to include some dependent headers that may be system specific + * for C-Ares */ +#define CARES_HAVE_SYS_TYPES_H +#define CARES_HAVE_SYS_SOCKET_H +/* #undef CARES_HAVE_WINDOWS_H */ +/* #undef CARES_HAVE_WS2TCPIP_H */ +/* #undef CARES_HAVE_WINSOCK2_H */ +/* #undef CARES_HAVE_WINDOWS_H */ +#define CARES_HAVE_ARPA_NAMESER_H +#define CARES_HAVE_ARPA_NAMESER_COMPAT_H + +#ifdef CARES_HAVE_SYS_TYPES_H +# include +#endif + +#ifdef CARES_HAVE_SYS_SOCKET_H +# include +#endif + +#ifdef CARES_HAVE_WINSOCK2_H +# include +#endif + +#ifdef CARES_HAVE_WS2TCPIP_H +# include +#endif + +#ifdef CARES_HAVE_WINDOWS_H +# include +#endif + + +typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t; +typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t; + +#endif /* __CARES_BUILD_H */ diff --git a/contrib/c-ares-cmake/freebsd/ares_config.h b/contrib/c-ares-cmake/freebsd/ares_config.h new file mode 100644 index 00000000000..a7836e0e802 --- /dev/null +++ b/contrib/c-ares-cmake/freebsd/ares_config.h @@ -0,0 +1,432 @@ +/* Generated from ares_config.h.cmake */ + +/* Define if building universal (internal helper macro) */ +#undef AC_APPLE_UNIVERSAL_BUILD + +/* define this if ares is built for a big endian system */ +#undef ARES_BIG_ENDIAN + +/* when building as static part of libcurl */ +#undef BUILDING_LIBCURL + +/* Defined for build that exposes internal static functions for testing. */ +#undef CARES_EXPOSE_STATICS + +/* Defined for build with symbol hiding. */ +#undef CARES_SYMBOL_HIDING + +/* Definition to make a library symbol externally visible. */ +#undef CARES_SYMBOL_SCOPE_EXTERN + +/* Use resolver library to configure cares */ +/* #undef CARES_USE_LIBRESOLV */ + +/* if a /etc/inet dir is being used */ +#undef ETC_INET + +/* Define to the type of arg 2 for gethostname. */ +#define GETHOSTNAME_TYPE_ARG2 size_t + +/* Define to the type qualifier of arg 1 for getnameinfo. */ +#define GETNAMEINFO_QUAL_ARG1 + +/* Define to the type of arg 1 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG1 struct sockaddr * + +/* Define to the type of arg 2 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG2 socklen_t + +/* Define to the type of args 4 and 6 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG46 socklen_t + +/* Define to the type of arg 7 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG7 int + +/* Specifies the number of arguments to getservbyport_r */ +#define GETSERVBYPORT_R_ARGS 6 + +/* Specifies the number of arguments to getservbyname_r */ +#define GETSERVBYNAME_R_ARGS 6 + +/* Define to 1 if you have AF_INET6. */ +#define HAVE_AF_INET6 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_COMPAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H + +/* Define to 1 if you have the `bitncmp' function. */ +/* #undef HAVE_BITNCMP */ + +/* Define to 1 if bool is an available type. */ +#define HAVE_BOOL_T + +/* Define to 1 if you have the clock_gettime function and monotonic timer. */ +#define HAVE_CLOCK_GETTIME_MONOTONIC + +/* Define to 1 if you have the closesocket function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the CloseSocket camel case function. */ +/* #undef HAVE_CLOSESOCKET_CAMEL */ + +/* Define to 1 if you have the connect function. */ +#define HAVE_CONNECT + +/* define if the compiler supports basic C++11 syntax */ +/* #undef HAVE_CXX11 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H + +/* Define to 1 if you have the fcntl function. */ +#define HAVE_FCNTL + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H + +/* Define to 1 if you have a working fcntl O_NONBLOCK function. */ +#define HAVE_FCNTL_O_NONBLOCK + +/* Define to 1 if you have the freeaddrinfo function. */ +#define HAVE_FREEADDRINFO + +/* Define to 1 if you have a working getaddrinfo function. */ +#define HAVE_GETADDRINFO + +/* Define to 1 if the getaddrinfo function is threadsafe. */ +#define HAVE_GETADDRINFO_THREADSAFE + +/* Define to 1 if you have the getenv function. */ +#define HAVE_GETENV + +/* Define to 1 if you have the gethostbyaddr function. */ +#define HAVE_GETHOSTBYADDR + +/* Define to 1 if you have the gethostbyname function. */ +#define HAVE_GETHOSTBYNAME + +/* Define to 1 if you have the gethostname function. */ +#define HAVE_GETHOSTNAME + +/* Define to 1 if you have the getnameinfo function. */ +#define HAVE_GETNAMEINFO + +/* Define to 1 if you have the getservbyport_r function. */ +#define HAVE_GETSERVBYPORT_R + +/* Define to 1 if you have the getservbyname_r function. */ +#define HAVE_GETSERVBYNAME_R + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY + +/* Define to 1 if you have the `if_indextoname' function. */ +#define HAVE_IF_INDEXTONAME + +/* Define to 1 if you have a IPv6 capable working inet_net_pton function. */ +/* #undef HAVE_INET_NET_PTON */ + +/* Define to 1 if you have a IPv6 capable working inet_ntop function. */ +#define HAVE_INET_NTOP + +/* Define to 1 if you have a IPv6 capable working inet_pton function. */ +#define HAVE_INET_PTON + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H + +/* Define to 1 if you have the ioctl function. */ +#define HAVE_IOCTL + +/* Define to 1 if you have the ioctlsocket function. */ +/* #undef HAVE_IOCTLSOCKET */ + +/* Define to 1 if you have the IoctlSocket camel case function. */ +/* #undef HAVE_IOCTLSOCKET_CAMEL */ + +/* Define to 1 if you have a working IoctlSocket camel case FIONBIO function. + */ +/* #undef HAVE_IOCTLSOCKET_CAMEL_FIONBIO */ + +/* Define to 1 if you have a working ioctlsocket FIONBIO function. */ +/* #undef HAVE_IOCTLSOCKET_FIONBIO */ + +/* Define to 1 if you have a working ioctl FIONBIO function. */ +#define HAVE_IOCTL_FIONBIO + +/* Define to 1 if you have a working ioctl SIOCGIFADDR function. */ +#define HAVE_IOCTL_SIOCGIFADDR + +/* Define to 1 if you have the `resolve' library (-lresolve). */ +/* #undef HAVE_LIBRESOLV */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H + +/* if your compiler supports LL */ +#define HAVE_LL + +/* Define to 1 if the compiler supports the 'long long' data type. */ +#define HAVE_LONGLONG + +/* Define to 1 if you have the malloc.h header file. */ +/* #undef HAVE_MALLOC_H */ + +/* Define to 1 if you have the memory.h header file. */ +#define HAVE_MEMORY_H + +/* Define to 1 if you have the MSG_NOSIGNAL flag. */ +#define HAVE_MSG_NOSIGNAL + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H + +/* Define to 1 if you have PF_INET6. */ +#define HAVE_PF_INET6 + +/* Define to 1 if you have the recv function. */ +#define HAVE_RECV + +/* Define to 1 if you have the recvfrom function. */ +#define HAVE_RECVFROM + +/* Define to 1 if you have the send function. */ +#define HAVE_SEND + +/* Define to 1 if you have the setsockopt function. */ +#define HAVE_SETSOCKOPT + +/* Define to 1 if you have a working setsockopt SO_NONBLOCK function. */ +/* #undef HAVE_SETSOCKOPT_SO_NONBLOCK */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SIGNAL_H + +/* Define to 1 if sig_atomic_t is an available typedef. */ +#define HAVE_SIG_ATOMIC_T + +/* Define to 1 if sig_atomic_t is already defined as volatile. */ +/* #undef HAVE_SIG_ATOMIC_T_VOLATILE */ + +/* Define to 1 if your struct sockaddr_in6 has sin6_scope_id. */ +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID + +/* Define to 1 if you have the socket function. */ +#define HAVE_SOCKET + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SOCKET_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDBOOL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H + +/* Define to 1 if you have the strcasecmp function. */ +#define HAVE_STRCASECMP + +/* Define to 1 if you have the strcmpi function. */ +/* #undef HAVE_STRCMPI */ + +/* Define to 1 if you have the strdup function. */ +#define HAVE_STRDUP + +/* Define to 1 if you have the stricmp function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H + +/* Define to 1 if you have the strncasecmp function. */ +#define HAVE_STRNCASECMP + +/* Define to 1 if you have the strncmpi function. */ +/* #undef HAVE_STRNCMPI */ + +/* Define to 1 if you have the strnicmp function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_STROPTS_H */ + +/* Define to 1 if you have struct addrinfo. */ +#define HAVE_STRUCT_ADDRINFO + +/* Define to 1 if you have struct in6_addr. */ +#define HAVE_STRUCT_IN6_ADDR + +/* Define to 1 if you have struct sockaddr_in6. */ +#define HAVE_STRUCT_SOCKADDR_IN6 + +/* if struct sockaddr_storage is defined */ +#define HAVE_STRUCT_SOCKADDR_STORAGE + +/* Define to 1 if you have the timeval struct. */ +#define HAVE_STRUCT_TIMEVAL + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H + +/* Define to 1 if you have the windows.h header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the winsock2.h header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the winsock.h header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the writev function. */ +#define HAVE_WRITEV + +/* Define to 1 if you have the ws2tcpip.h header file. */ +/* #undef HAVE_WS2TCPIP_H */ + +/* Define to 1 if you have the __system_property_get function */ +#define HAVE___SYSTEM_PROPERTY_GET + +/* Define to 1 if you need the malloc.h header file even with stdlib.h */ +/* #undef NEED_MALLOC_H */ + +/* Define to 1 if you need the memory.h header file even with stdlib.h */ +/* #undef NEED_MEMORY_H */ + +/* a suitable file/device to read random data from */ +#define CARES_RANDOM_FILE "/dev/urandom" + +/* Define to the type qualifier pointed by arg 5 for recvfrom. */ +#define RECVFROM_QUAL_ARG5 + +/* Define to the type of arg 1 for recvfrom. */ +#define RECVFROM_TYPE_ARG1 int + +/* Define to the type pointed by arg 2 for recvfrom. */ +#define RECVFROM_TYPE_ARG2 void * + +/* Define to 1 if the type pointed by arg 2 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG2_IS_VOID 0 + +/* Define to the type of arg 3 for recvfrom. */ +#define RECVFROM_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recvfrom. */ +#define RECVFROM_TYPE_ARG4 int + +/* Define to the type pointed by arg 5 for recvfrom. */ +#define RECVFROM_TYPE_ARG5 struct sockaddr * + +/* Define to 1 if the type pointed by arg 5 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG5_IS_VOID 0 + +/* Define to the type pointed by arg 6 for recvfrom. */ +#define RECVFROM_TYPE_ARG6 socklen_t * + +/* Define to 1 if the type pointed by arg 6 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG6_IS_VOID 0 + +/* Define to the function return type for recvfrom. */ +#define RECVFROM_TYPE_RETV ssize_t + +/* Define to the type of arg 1 for recv. */ +#define RECV_TYPE_ARG1 int + +/* Define to the type of arg 2 for recv. */ +#define RECV_TYPE_ARG2 void * + +/* Define to the type of arg 3 for recv. */ +#define RECV_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recv. */ +#define RECV_TYPE_ARG4 int + +/* Define to the function return type for recv. */ +#define RECV_TYPE_RETV ssize_t + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE + +/* Define to the type qualifier of arg 2 for send. */ +#define SEND_QUAL_ARG2 + +/* Define to the type of arg 1 for send. */ +#define SEND_TYPE_ARG1 int + +/* Define to the type of arg 2 for send. */ +#define SEND_TYPE_ARG2 void * + +/* Define to the type of arg 3 for send. */ +#define SEND_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for send. */ +#define SEND_TYPE_ARG4 int + +/* Define to the function return type for send. */ +#define SEND_TYPE_RETV ssize_t + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME + +/* Define to disable non-blocking sockets. */ +#undef USE_BLOCKING_SOCKETS + +/* Define to avoid automatic inclusion of winsock.h */ +#undef WIN32_LEAN_AND_MEAN + +/* Type to use in place of in_addr_t when system does not provide it. */ +#undef in_addr_t + diff --git a/contrib/c-ares-cmake/linux/ares_build.h b/contrib/c-ares-cmake/linux/ares_build.h new file mode 100644 index 00000000000..bf7402e7997 --- /dev/null +++ b/contrib/c-ares-cmake/linux/ares_build.h @@ -0,0 +1,43 @@ +#ifndef __CARES_BUILD_H +#define __CARES_BUILD_H + +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Prefix names with CARES_ to make sure they don't conflict with other config.h + * files. We need to include some dependent headers that may be system specific + * for C-Ares */ +#define CARES_HAVE_SYS_TYPES_H +#define CARES_HAVE_SYS_SOCKET_H +/* #undef CARES_HAVE_WINDOWS_H */ +/* #undef CARES_HAVE_WS2TCPIP_H */ +/* #undef CARES_HAVE_WINSOCK2_H */ +/* #undef CARES_HAVE_WINDOWS_H */ +#define CARES_HAVE_ARPA_NAMESER_H +#define CARES_HAVE_ARPA_NAMESER_COMPAT_H + +#ifdef CARES_HAVE_SYS_TYPES_H +# include +#endif + +#ifdef CARES_HAVE_SYS_SOCKET_H +# include +#endif + +#ifdef CARES_HAVE_WINSOCK2_H +# include +#endif + +#ifdef CARES_HAVE_WS2TCPIP_H +# include +#endif + +#ifdef CARES_HAVE_WINDOWS_H +# include +#endif + + +typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t; +typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t; + +#endif /* __CARES_BUILD_H */ diff --git a/contrib/c-ares-cmake/linux/ares_config.h b/contrib/c-ares-cmake/linux/ares_config.h new file mode 100644 index 00000000000..e0ebf86e842 --- /dev/null +++ b/contrib/c-ares-cmake/linux/ares_config.h @@ -0,0 +1,432 @@ +/* Generated from ares_config.h.cmake */ + +/* Define if building universal (internal helper macro) */ +#undef AC_APPLE_UNIVERSAL_BUILD + +/* define this if ares is built for a big endian system */ +#undef ARES_BIG_ENDIAN + +/* when building as static part of libcurl */ +#undef BUILDING_LIBCURL + +/* Defined for build that exposes internal static functions for testing. */ +#undef CARES_EXPOSE_STATICS + +/* Defined for build with symbol hiding. */ +#undef CARES_SYMBOL_HIDING + +/* Definition to make a library symbol externally visible. */ +#undef CARES_SYMBOL_SCOPE_EXTERN + +/* Use resolver library to configure cares */ +/* #undef CARES_USE_LIBRESOLV */ + +/* if a /etc/inet dir is being used */ +#undef ETC_INET + +/* Define to the type of arg 2 for gethostname. */ +#define GETHOSTNAME_TYPE_ARG2 size_t + +/* Define to the type qualifier of arg 1 for getnameinfo. */ +#define GETNAMEINFO_QUAL_ARG1 + +/* Define to the type of arg 1 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG1 struct sockaddr * + +/* Define to the type of arg 2 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG2 socklen_t + +/* Define to the type of args 4 and 6 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG46 socklen_t + +/* Define to the type of arg 7 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG7 int + +/* Specifies the number of arguments to getservbyport_r */ +#define GETSERVBYPORT_R_ARGS 6 + +/* Specifies the number of arguments to getservbyname_r */ +#define GETSERVBYNAME_R_ARGS 6 + +/* Define to 1 if you have AF_INET6. */ +#define HAVE_AF_INET6 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_COMPAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H + +/* Define to 1 if you have the `bitncmp' function. */ +/* #undef HAVE_BITNCMP */ + +/* Define to 1 if bool is an available type. */ +#define HAVE_BOOL_T + +/* Define to 1 if you have the clock_gettime function and monotonic timer. */ +#define HAVE_CLOCK_GETTIME_MONOTONIC + +/* Define to 1 if you have the closesocket function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the CloseSocket camel case function. */ +/* #undef HAVE_CLOSESOCKET_CAMEL */ + +/* Define to 1 if you have the connect function. */ +#define HAVE_CONNECT + +/* define if the compiler supports basic C++11 syntax */ +/* #undef HAVE_CXX11 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H + +/* Define to 1 if you have the fcntl function. */ +#define HAVE_FCNTL + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H + +/* Define to 1 if you have a working fcntl O_NONBLOCK function. */ +#define HAVE_FCNTL_O_NONBLOCK + +/* Define to 1 if you have the freeaddrinfo function. */ +#define HAVE_FREEADDRINFO + +/* Define to 1 if you have a working getaddrinfo function. */ +#define HAVE_GETADDRINFO + +/* Define to 1 if the getaddrinfo function is threadsafe. */ +/* #undef HAVE_GETADDRINFO_THREADSAFE */ + +/* Define to 1 if you have the getenv function. */ +#define HAVE_GETENV + +/* Define to 1 if you have the gethostbyaddr function. */ +#define HAVE_GETHOSTBYADDR + +/* Define to 1 if you have the gethostbyname function. */ +#define HAVE_GETHOSTBYNAME + +/* Define to 1 if you have the gethostname function. */ +#define HAVE_GETHOSTNAME + +/* Define to 1 if you have the getnameinfo function. */ +#define HAVE_GETNAMEINFO + +/* Define to 1 if you have the getservbyport_r function. */ +#define HAVE_GETSERVBYPORT_R + +/* Define to 1 if you have the getservbyname_r function. */ +#define HAVE_GETSERVBYNAME_R + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY + +/* Define to 1 if you have the `if_indextoname' function. */ +#define HAVE_IF_INDEXTONAME + +/* Define to 1 if you have a IPv6 capable working inet_net_pton function. */ +/* #undef HAVE_INET_NET_PTON */ + +/* Define to 1 if you have a IPv6 capable working inet_ntop function. */ +#define HAVE_INET_NTOP + +/* Define to 1 if you have a IPv6 capable working inet_pton function. */ +#define HAVE_INET_PTON + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H + +/* Define to 1 if you have the ioctl function. */ +#define HAVE_IOCTL + +/* Define to 1 if you have the ioctlsocket function. */ +/* #undef HAVE_IOCTLSOCKET */ + +/* Define to 1 if you have the IoctlSocket camel case function. */ +/* #undef HAVE_IOCTLSOCKET_CAMEL */ + +/* Define to 1 if you have a working IoctlSocket camel case FIONBIO function. + */ +/* #undef HAVE_IOCTLSOCKET_CAMEL_FIONBIO */ + +/* Define to 1 if you have a working ioctlsocket FIONBIO function. */ +/* #undef HAVE_IOCTLSOCKET_FIONBIO */ + +/* Define to 1 if you have a working ioctl FIONBIO function. */ +#define HAVE_IOCTL_FIONBIO + +/* Define to 1 if you have a working ioctl SIOCGIFADDR function. */ +#define HAVE_IOCTL_SIOCGIFADDR + +/* Define to 1 if you have the `resolve' library (-lresolve). */ +/* #undef HAVE_LIBRESOLV */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H + +/* if your compiler supports LL */ +#define HAVE_LL + +/* Define to 1 if the compiler supports the 'long long' data type. */ +#define HAVE_LONGLONG + +/* Define to 1 if you have the malloc.h header file. */ +#define HAVE_MALLOC_H + +/* Define to 1 if you have the memory.h header file. */ +#define HAVE_MEMORY_H + +/* Define to 1 if you have the MSG_NOSIGNAL flag. */ +#define HAVE_MSG_NOSIGNAL + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H + +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H + +/* Define to 1 if you have PF_INET6. */ +#define HAVE_PF_INET6 + +/* Define to 1 if you have the recv function. */ +#define HAVE_RECV + +/* Define to 1 if you have the recvfrom function. */ +#define HAVE_RECVFROM + +/* Define to 1 if you have the send function. */ +#define HAVE_SEND + +/* Define to 1 if you have the setsockopt function. */ +#define HAVE_SETSOCKOPT + +/* Define to 1 if you have a working setsockopt SO_NONBLOCK function. */ +/* #undef HAVE_SETSOCKOPT_SO_NONBLOCK */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SIGNAL_H + +/* Define to 1 if sig_atomic_t is an available typedef. */ +#define HAVE_SIG_ATOMIC_T + +/* Define to 1 if sig_atomic_t is already defined as volatile. */ +/* #undef HAVE_SIG_ATOMIC_T_VOLATILE */ + +/* Define to 1 if your struct sockaddr_in6 has sin6_scope_id. */ +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID + +/* Define to 1 if you have the socket function. */ +#define HAVE_SOCKET + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SOCKET_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDBOOL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H + +/* Define to 1 if you have the strcasecmp function. */ +#define HAVE_STRCASECMP + +/* Define to 1 if you have the strcmpi function. */ +/* #undef HAVE_STRCMPI */ + +/* Define to 1 if you have the strdup function. */ +#define HAVE_STRDUP + +/* Define to 1 if you have the stricmp function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H + +/* Define to 1 if you have the strncasecmp function. */ +#define HAVE_STRNCASECMP + +/* Define to 1 if you have the strncmpi function. */ +/* #undef HAVE_STRNCMPI */ + +/* Define to 1 if you have the strnicmp function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STROPTS_H + +/* Define to 1 if you have struct addrinfo. */ +#define HAVE_STRUCT_ADDRINFO + +/* Define to 1 if you have struct in6_addr. */ +#define HAVE_STRUCT_IN6_ADDR + +/* Define to 1 if you have struct sockaddr_in6. */ +#define HAVE_STRUCT_SOCKADDR_IN6 + +/* if struct sockaddr_storage is defined */ +#define HAVE_STRUCT_SOCKADDR_STORAGE + +/* Define to 1 if you have the timeval struct. */ +#define HAVE_STRUCT_TIMEVAL + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H + +/* Define to 1 if you have the windows.h header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the winsock2.h header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the winsock.h header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the writev function. */ +#define HAVE_WRITEV + +/* Define to 1 if you have the ws2tcpip.h header file. */ +/* #undef HAVE_WS2TCPIP_H */ + +/* Define to 1 if you have the __system_property_get function */ +#define HAVE___SYSTEM_PROPERTY_GET + +/* Define to 1 if you need the malloc.h header file even with stdlib.h */ +/* #undef NEED_MALLOC_H */ + +/* Define to 1 if you need the memory.h header file even with stdlib.h */ +/* #undef NEED_MEMORY_H */ + +/* a suitable file/device to read random data from */ +#define CARES_RANDOM_FILE "/dev/urandom" + +/* Define to the type qualifier pointed by arg 5 for recvfrom. */ +#define RECVFROM_QUAL_ARG5 + +/* Define to the type of arg 1 for recvfrom. */ +#define RECVFROM_TYPE_ARG1 int + +/* Define to the type pointed by arg 2 for recvfrom. */ +#define RECVFROM_TYPE_ARG2 void * + +/* Define to 1 if the type pointed by arg 2 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG2_IS_VOID 0 + +/* Define to the type of arg 3 for recvfrom. */ +#define RECVFROM_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recvfrom. */ +#define RECVFROM_TYPE_ARG4 int + +/* Define to the type pointed by arg 5 for recvfrom. */ +#define RECVFROM_TYPE_ARG5 struct sockaddr * + +/* Define to 1 if the type pointed by arg 5 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG5_IS_VOID 0 + +/* Define to the type pointed by arg 6 for recvfrom. */ +#define RECVFROM_TYPE_ARG6 socklen_t * + +/* Define to 1 if the type pointed by arg 6 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG6_IS_VOID 0 + +/* Define to the function return type for recvfrom. */ +#define RECVFROM_TYPE_RETV ssize_t + +/* Define to the type of arg 1 for recv. */ +#define RECV_TYPE_ARG1 int + +/* Define to the type of arg 2 for recv. */ +#define RECV_TYPE_ARG2 void * + +/* Define to the type of arg 3 for recv. */ +#define RECV_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recv. */ +#define RECV_TYPE_ARG4 int + +/* Define to the function return type for recv. */ +#define RECV_TYPE_RETV ssize_t + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE + +/* Define to the type qualifier of arg 2 for send. */ +#define SEND_QUAL_ARG2 + +/* Define to the type of arg 1 for send. */ +#define SEND_TYPE_ARG1 int + +/* Define to the type of arg 2 for send. */ +#define SEND_TYPE_ARG2 void * + +/* Define to the type of arg 3 for send. */ +#define SEND_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for send. */ +#define SEND_TYPE_ARG4 int + +/* Define to the function return type for send. */ +#define SEND_TYPE_RETV ssize_t + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME + +/* Define to disable non-blocking sockets. */ +#undef USE_BLOCKING_SOCKETS + +/* Define to avoid automatic inclusion of winsock.h */ +#undef WIN32_LEAN_AND_MEAN + +/* Type to use in place of in_addr_t when system does not provide it. */ +#undef in_addr_t + diff --git a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index ff97d297d8f..e08a2bed2ec 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -415,7 +415,7 @@ /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. */ -#define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE +/* #undef JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE */ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ diff --git a/contrib/krb5 b/contrib/krb5 index d879821c7a4..b89e20367b0 160000 --- a/contrib/krb5 +++ b/contrib/krb5 @@ -1 +1 @@ -Subproject commit d879821c7a4c70b0c3ad739d9951d1a2b1903df7 +Subproject commit b89e20367b074bd02dd118a6534099b21e88b3c3 diff --git a/contrib/krb5-cmake/autoconf_linux.h b/contrib/krb5-cmake/autoconf_linux.h index 7b71d962d9a..54951f866a5 100644 --- a/contrib/krb5-cmake/autoconf_linux.h +++ b/contrib/krb5-cmake/autoconf_linux.h @@ -440,7 +440,9 @@ #define HAVE_STRERROR 1 /* Define to 1 if you have the `strerror_r' function. */ +#ifndef USE_MUSL #define HAVE_STRERROR_R 1 +#endif /* Define to 1 if you have the header file. */ #define HAVE_STRINGS_H 1 diff --git a/contrib/libcpuid b/contrib/libcpuid index 8db3b8d2d32..503083acb77 160000 --- a/contrib/libcpuid +++ b/contrib/libcpuid @@ -1 +1 @@ -Subproject commit 8db3b8d2d32d22437f063ce692a1b9bb15e42d18 +Subproject commit 503083acb77edf9fbce22a05826307dff2ce96e6 diff --git a/contrib/libpq-cmake/CMakeLists.txt b/contrib/libpq-cmake/CMakeLists.txt index 280c0381393..91326422b43 100644 --- a/contrib/libpq-cmake/CMakeLists.txt +++ b/contrib/libpq-cmake/CMakeLists.txt @@ -63,6 +63,13 @@ target_include_directories (_libpq SYSTEM PUBLIC ${LIBPQ_SOURCE_DIR}) target_include_directories (_libpq SYSTEM PUBLIC "${LIBPQ_SOURCE_DIR}/include") target_include_directories (_libpq SYSTEM PRIVATE "${LIBPQ_SOURCE_DIR}/configs") +# NOTE: this is a dirty hack to avoid and instead pg_config.h should be shipped +# for different OS'es like for jemalloc, not one generic for all OS'es like +# now. +if (OS_DARWIN OR OS_FREEBSD OR USE_MUSL) + target_compile_definitions(_libpq PRIVATE -DSTRERROR_R_INT=1) +endif() + target_link_libraries (_libpq PRIVATE OpenSSL::SSL) add_library(ch_contrib::libpq ALIAS _libpq) diff --git a/contrib/librdkafka b/contrib/librdkafka index ff32b4e9eea..6f3b483426a 160000 --- a/contrib/librdkafka +++ b/contrib/librdkafka @@ -1 +1 @@ -Subproject commit ff32b4e9eeafd0b276f010ee969179e4e9e6d0b2 +Subproject commit 6f3b483426a8c8ec950e27e446bec175cf8b553f diff --git a/contrib/llvm b/contrib/llvm index 20607e61728..0db5bf5bd24 160000 --- a/contrib/llvm +++ b/contrib/llvm @@ -1 +1 @@ -Subproject commit 20607e61728e97c969e536644c3c0c1bb1a50672 +Subproject commit 0db5bf5bd2452cd8f1283a1fcdc04845af705bfc diff --git a/contrib/replxx b/contrib/replxx index 3fd0e3c9364..5d04501f93a 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit 3fd0e3c9364a589447453d9906d854ebd8d385c5 +Subproject commit 5d04501f93a4fb7f0bb8b73b8f614bc986f9e25b diff --git a/contrib/sentry-native b/contrib/sentry-native index f431047ac8d..ae10fb8c224 160000 --- a/contrib/sentry-native +++ b/contrib/sentry-native @@ -1 +1 @@ -Subproject commit f431047ac8da13179c488018dddf1c0d0771a997 +Subproject commit ae10fb8c224c3f41571446e1ed7fd57b9e5e366b diff --git a/contrib/vectorscan b/contrib/vectorscan index 73695e419c2..f6250ae3e5a 160000 --- a/contrib/vectorscan +++ b/contrib/vectorscan @@ -1 +1 @@ -Subproject commit 73695e419c27af7fe2a099c7aa57931cc02aea5d +Subproject commit f6250ae3e5a3085000239313ad0689cc1e00cdc2 diff --git a/contrib/vectorscan-cmake/CMakeLists.txt b/contrib/vectorscan-cmake/CMakeLists.txt index 828f2a17df2..d6c626c1612 100644 --- a/contrib/vectorscan-cmake/CMakeLists.txt +++ b/contrib/vectorscan-cmake/CMakeLists.txt @@ -304,7 +304,7 @@ target_include_directories (_vectorscan SYSTEM PUBLIC "${LIBRARY_DIR}/src") # Please regenerate these files if you update vectorscan. if (ARCH_AMD64) - target_include_directories (_vectorscan PRIVATE x86_64) + target_include_directories (_vectorscan PRIVATE amd64) endif () if (ARCH_AARCH64) diff --git a/contrib/vectorscan-cmake/x86_64/config.h b/contrib/vectorscan-cmake/amd64/config.h similarity index 100% rename from contrib/vectorscan-cmake/x86_64/config.h rename to contrib/vectorscan-cmake/amd64/config.h diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt index 371a07dd31a..aa067ba37e0 100644 --- a/contrib/zlib-ng-cmake/CMakeLists.txt +++ b/contrib/zlib-ng-cmake/CMakeLists.txt @@ -2,8 +2,10 @@ set (SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/zlib-ng) add_definitions(-DZLIB_COMPAT) add_definitions(-DWITH_GZFILEOP) -add_definitions(-DUNALIGNED_OK) -add_definitions(-DUNALIGNED64_OK) +if(NOT ARCH_S390X) + add_definitions(-DUNALIGNED_OK) + add_definitions(-DUNALIGNED64_OK) +endif() set (HAVE_UNISTD_H 1) add_definitions(-D_LARGEFILE64_SOURCE=1 -D__USE_LARGEFILE64) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index b9b0c5c2c6c..db55c950241 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -46,7 +46,7 @@ RUN apt-get install binutils-riscv64-linux-gnu # Architecture of the image when BuildKit/buildx is used ARG TARGETARCH -ARG NFPM_VERSION=2.16.0 +ARG NFPM_VERSION=2.18.1 RUN arch=${TARGETARCH:-amd64} \ && curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \ @@ -67,24 +67,5 @@ ENV GOCACHE=/workdir/ RUN mkdir /workdir && chmod 777 /workdir WORKDIR /workdir -# NOTE: thread sanitizer is broken in clang-14, we have to build it with clang-15 -# https://github.com/ClickHouse/ClickHouse/pull/39450 -# https://github.com/google/sanitizers/issues/1540 -# https://github.com/google/sanitizers/issues/1552 - -RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ - && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-15 main" >> \ - /etc/apt/sources.list.d/clang.list \ - && apt-get update \ - && apt-get install \ - clang-15 \ - llvm-15 \ - clang-tidy-15 \ - --yes --no-install-recommends \ - && apt-get clean - -# for external_symbolizer_path -RUN ln -s /usr/bin/llvm-symbolizer-15 /usr/bin/llvm-symbolizer - COPY build.sh / CMD ["bash", "-c", "/build.sh 2>&1"] diff --git a/docker/packager/packager b/docker/packager/packager index 591262959b4..9da787e9006 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -339,17 +339,16 @@ if __name__ == "__main__": parser.add_argument( "--compiler", choices=( - "clang-15", # For TSAN builds, see #39450 - "clang-14", - "clang-14-darwin", - "clang-14-darwin-aarch64", - "clang-14-aarch64", - "clang-14-ppc64le", - "clang-14-amd64sse2", - "clang-14-freebsd", + "clang-15", + "clang-15-darwin", + "clang-15-darwin-aarch64", + "clang-15-aarch64", + "clang-15-ppc64le", + "clang-15-amd64sse2", + "clang-15-freebsd", "gcc-11", ), - default="clang-14", + default="clang-15", help="a compiler to use", ) parser.add_argument( diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index b01dba1e22f..1a672f30a74 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="20.9.3.45" +ARG VERSION="22.8.5.29" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index f4102a6ccaf..db76a9fab1d 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION=22.6.1.* +ARG VERSION="22.8.5.29" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index 16372230d91..dfd1000d809 100755 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -38,6 +38,7 @@ FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_ # There could be many disks declared in config readarray -t DISKS_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.path' || true) +readarray -t DISKS_METADATA_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.metadata_path' || true) CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" @@ -50,7 +51,8 @@ for dir in "$DATA_DIR" \ "$TMP_DIR" \ "$USER_PATH" \ "$FORMAT_SCHEMA_PATH" \ - "${DISKS_PATHS[@]}" + "${DISKS_PATHS[@]}" \ + "${DISKS_METADATA_PATHS[@]}" do # check if variable not empty [ -z "$dir" ] && continue @@ -106,7 +108,7 @@ if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then # port is needed to check if clickhouse-server is ready for connections HTTP_PORT="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=http_port)" HTTPS_PORT="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=https_port)" - + if [ -n "$HTTP_PORT" ]; then URL="http://127.0.0.1:$HTTP_PORT/ping" else diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 43cfca1fdfc..4e42fce1a1d 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -16,11 +16,10 @@ RUN apt-get update \ # and MEMORY_LIMIT_EXCEEDED exceptions in Functional tests (total memory limit in Functional tests is ~55.24 GiB). # TSAN will flush shadow memory when reaching this limit. # It may cause false-negatives, but it's better than OOM. -RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7 memory_limit_mb=46080'" >> /etc/environment; \ - echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ - echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'" >> /etc/environment; \ - echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment; \ - ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; +RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7 memory_limit_mb=46080'" >> /etc/environment +RUN echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment +RUN echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'" >> /etc/environment +RUN echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment # Sanitizer options for current shell (not current, but the one that will be spawned on "docker run") # (but w/o verbosity for TSAN, otherwise test.reference will not match) ENV TSAN_OPTIONS='halt_on_error=1 history_size=7 memory_limit_mb=46080' diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index c7aed618f6a..ceed93c3ac7 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -8,16 +8,41 @@ FROM clickhouse/binary-builder:$FROM_TAG ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-14 libllvm14 libclang-14-dev libmlir-14-dev +RUN apt-get update && apt-get --yes --allow-unauthenticated install libclang-${LLVM_VERSION}-dev libmlir-${LLVM_VERSION}-dev + +# libclang-15-dev does not contain proper symlink: +# +# This is what cmake will search for: +# +# # readlink -f /usr/lib/llvm-15/lib/libclang-15.so.1 +# /usr/lib/x86_64-linux-gnu/libclang-15.so.1 +# +# This is what exists: +# +# # ls -l /usr/lib/x86_64-linux-gnu/libclang-15* +# lrwxrwxrwx 1 root root 16 Sep 5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so -> libclang-15.so.1 +# lrwxrwxrwx 1 root root 21 Sep 5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so.15 -> libclang-15.so.15.0.0 +# -rw-r--r-- 1 root root 31835760 Sep 5 13:31 /usr/lib/x86_64-linux-gnu/libclang-15.so.15.0.0 +# +ARG TARGETARCH +RUN arch=${TARGETARCH:-amd64} \ + && case $arch in \ + amd64) rarch=x86_64 ;; \ + arm64) rarch=aarch64 ;; \ + *) exit 1 ;; \ + esac \ + && ln -rsf /usr/lib/$rarch-linux-gnu/libclang-15.so.15 /usr/lib/$rarch-linux-gnu/libclang-15.so.1 # repo versions doesn't work correctly with C++17 # also we push reports to s3, so we add index.html to subfolder urls # https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b # TODO: remove branch in a few weeks after merge, e.g. in May or June 2022 -RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser --branch llvm-14 \ +# +# FIXME: update location of a repo +RUN git clone https://github.com/azat/woboq_codebrowser --branch llvm-15 \ && cd woboq_codebrowser \ - && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-14 -DCMAKE_C_COMPILER=clang-14 \ - && make -j \ + && cmake . -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=clang-${LLVM_VERSION} \ + && ninja \ && cd .. \ && rm -rf woboq_codebrowser @@ -32,7 +57,7 @@ ENV SHA=nosha ENV DATA="https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data" CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \ - cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-14 -DCMAKE_C_COMPILER=/usr/bin/clang-14 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \ + cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-${LLVM_VERSION} -DCMAKE_C_COMPILER=/usr/bin/clang-${LLVM_VERSION} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \ mkdir -p $HTML_RESULT_DIRECTORY && \ $CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \ cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\ diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 93e38260395..bab87865b42 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -19,7 +19,7 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "$script_dir" repo_dir=ch -BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-14_debug_none_unsplitted_disable_False_binary"} +BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-15_debug_none_unsplitted_disable_False_binary"} BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"} function clone diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index bcaa064fe4f..5ae880ddf36 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -28,10 +28,9 @@ done set -e # cleanup for retry run if volume is not recreated -# shellcheck disable=SC2046 { - docker ps -aq | xargs -r docker kill || true - docker ps -aq | xargs -r docker rm || true + docker ps --all --quiet | xargs --no-run-if-empty docker kill || true + docker ps --all --quiet | xargs --no-run-if-empty docker rm || true } echo "Start tests" diff --git a/docker/test/keeper-jepsen/run.sh b/docker/test/keeper-jepsen/run.sh index c43e6b2c54d..adf99c029a9 100644 --- a/docker/test/keeper-jepsen/run.sh +++ b/docker/test/keeper-jepsen/run.sh @@ -2,7 +2,7 @@ set -euo pipefail -CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-14_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"} +CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-15_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"} CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""} diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index d3d7084f37f..b0b5ebdb2e2 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -61,7 +61,7 @@ function configure cp -rv right/config left ||: # Start a temporary server to rename the tables - while pkill clickhouse-serv; do echo . ; sleep 1 ; done + while pkill -f clickhouse-serv ; do echo . ; sleep 1 ; done echo all killed set -m # Spawn temporary in its own process groups @@ -88,7 +88,7 @@ function configure clickhouse-client --port $LEFT_SERVER_PORT --query "create database test" ||: clickhouse-client --port $LEFT_SERVER_PORT --query "rename table datasets.hits_v1 to test.hits" ||: - while pkill clickhouse-serv; do echo . ; sleep 1 ; done + while pkill -f clickhouse-serv ; do echo . ; sleep 1 ; done echo all killed # Make copies of the original db for both servers. Use hardlinks instead @@ -106,7 +106,7 @@ function configure function restart { - while pkill clickhouse-serv; do echo . ; sleep 1 ; done + while pkill -f clickhouse-serv ; do echo . ; sleep 1 ; done echo all killed # Change the jemalloc settings here. @@ -1400,7 +1400,7 @@ case "$stage" in while env kill -- -$watchdog_pid ; do sleep 1; done # Stop the servers to free memory for the subsequent query analysis. - while pkill clickhouse-serv; do echo . ; sleep 1 ; done + while pkill -f clickhouse-serv ; do echo . ; sleep 1 ; done echo Servers stopped. ;& "analyze_queries") diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 72b0eb5bda1..8151ec9d5e4 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -179,17 +179,17 @@ pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhous # for files >64MB, we want this files to be compressed explicitly for table in query_log zookeeper_log trace_log transactions_info_log do - clickhouse-local --path /var/lib/clickhouse/ -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz ||: + clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz ||: if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then - clickhouse-local --path /var/lib/clickhouse1/ -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.1.tsv.gz ||: - clickhouse-local --path /var/lib/clickhouse2/ -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.2.tsv.gz ||: + clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.1.tsv.gz ||: + clickhouse-local --path /var/lib/clickhouse2/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.2.tsv.gz ||: fi done # Also export trace log in flamegraph-friendly format. for trace_type in CPU Memory Real do - clickhouse-local --path /var/lib/clickhouse/ -q " + clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q " select arrayStringConcat((arrayMap(x -> concat(splitByChar('/', addressToLine(x))[-1], '#', demangle(addressToSymbol(x)) ), trace)), ';') AS stack, count(*) AS samples diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index ac7de9c07a2..25bc64261f3 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -370,6 +370,7 @@ else # Avoid "Setting s3_check_objects_after_upload is neither a builtin setting..." rm -f /etc/clickhouse-server/users.d/enable_blobs_check.xml ||: + rm -f /etc/clickhouse-server/users.d/marks.xml ||: # Remove s3 related configs to avoid "there is no disk type `cache`" rm -f /etc/clickhouse-server/config.d/storage_conf.xml ||: diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index b891b71492c..57880bfc1d6 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -5,7 +5,7 @@ FROM ubuntu:20.04 ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=15 RUN apt-get update \ && apt-get install \ @@ -56,6 +56,8 @@ RUN apt-get update \ # This symlink required by gcc to find lld compiler RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld +# for external_symbolizer_path +RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer ARG CCACHE_VERSION=4.6.1 RUN mkdir /tmp/ccache \ diff --git a/docs/changelogs/v22.6.8.35-stable.md b/docs/changelogs/v22.6.8.35-stable.md new file mode 100644 index 00000000000..ffc5e03e489 --- /dev/null +++ b/docs/changelogs/v22.6.8.35-stable.md @@ -0,0 +1,34 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.6.8.35-stable (b91dc59a565) FIXME as compared to v22.6.7.7-stable (8eae2af3b9a) + +#### New Feature +* Backported in [#40868](https://github.com/ClickHouse/ClickHouse/issues/40868): Add setting to disable limit on kafka_num_consumers. Closes [#40331](https://github.com/ClickHouse/ClickHouse/issues/40331). [#40670](https://github.com/ClickHouse/ClickHouse/pull/40670) ([Kruglov Pavel](https://github.com/Avogar)). + +#### Bug Fix +* Backported in [#41274](https://github.com/ClickHouse/ClickHouse/issues/41274): Fix memory safety issues with functions `encrypt` and `contingency` if Array of Nullable is used as an argument. This fixes [#41004](https://github.com/ClickHouse/ClickHouse/issues/41004). [#40195](https://github.com/ClickHouse/ClickHouse/pull/40195) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#41282](https://github.com/ClickHouse/ClickHouse/issues/41282): Fix unused unknown columns introduced by WITH statement. This fixes [#37812](https://github.com/ClickHouse/ClickHouse/issues/37812) . [#39131](https://github.com/ClickHouse/ClickHouse/pull/39131) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#40905](https://github.com/ClickHouse/ClickHouse/issues/40905): Fix potential deadlock in WriteBufferFromS3 during task scheduling failure. [#40070](https://github.com/ClickHouse/ClickHouse/pull/40070) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#40864](https://github.com/ClickHouse/ClickHouse/issues/40864): - Fix crash while parsing values of type `Object` that contains arrays of variadic dimension. [#40483](https://github.com/ClickHouse/ClickHouse/pull/40483) ([Duc Canh Le](https://github.com/canhld94)). +* Backported in [#40803](https://github.com/ClickHouse/ClickHouse/issues/40803): During insertion of a new query to the `ProcessList` allocations happen. If we reach the memory limit during these allocations we can not use `OvercommitTracker`, because `ProcessList::mutex` is already acquired. Fixes [#40611](https://github.com/ClickHouse/ClickHouse/issues/40611). [#40677](https://github.com/ClickHouse/ClickHouse/pull/40677) ([Dmitry Novik](https://github.com/novikd)). +* Backported in [#40891](https://github.com/ClickHouse/ClickHouse/issues/40891): Fix memory leak while pushing to MVs w/o query context (from Kafka/...). [#40732](https://github.com/ClickHouse/ClickHouse/pull/40732) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#41133](https://github.com/ClickHouse/ClickHouse/issues/41133): Fix access rights for `DESCRIBE TABLE url()` and some other `DESCRIBE TABLE ()`. [#40975](https://github.com/ClickHouse/ClickHouse/pull/40975) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#41360](https://github.com/ClickHouse/ClickHouse/issues/41360): Fix incorrect logical error `Expected relative path` in disk object storage. Related to [#41246](https://github.com/ClickHouse/ClickHouse/issues/41246). [#41297](https://github.com/ClickHouse/ClickHouse/pull/41297) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#41357](https://github.com/ClickHouse/ClickHouse/issues/41357): Add column type check before UUID insertion in MsgPack format. [#41309](https://github.com/ClickHouse/ClickHouse/pull/41309) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* use ROBOT_CLICKHOUSE_COMMIT_TOKEN for create-pull-request [#40067](https://github.com/ClickHouse/ClickHouse/pull/40067) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* use input token instead of env var [#40421](https://github.com/ClickHouse/ClickHouse/pull/40421) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Migrate artifactory [#40831](https://github.com/ClickHouse/ClickHouse/pull/40831) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Docker server version [#41256](https://github.com/ClickHouse/ClickHouse/pull/41256) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Increase open files limit [#41345](https://github.com/ClickHouse/ClickHouse/pull/41345) ([Eugene Konkov](https://github.com/ekonkov)). + diff --git a/docs/changelogs/v22.8.5.29-lts.md b/docs/changelogs/v22.8.5.29-lts.md new file mode 100644 index 00000000000..0ce13b7c36e --- /dev/null +++ b/docs/changelogs/v22.8.5.29-lts.md @@ -0,0 +1,34 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.8.5.29-lts (74ffb843807) FIXME as compared to v22.8.4.7-lts (baad27bcd2f) + +#### New Feature +* Backported in [#40870](https://github.com/ClickHouse/ClickHouse/issues/40870): Add setting to disable limit on kafka_num_consumers. Closes [#40331](https://github.com/ClickHouse/ClickHouse/issues/40331). [#40670](https://github.com/ClickHouse/ClickHouse/pull/40670) ([Kruglov Pavel](https://github.com/Avogar)). + +#### Improvement +* Backported in [#40817](https://github.com/ClickHouse/ClickHouse/issues/40817): The setting `show_addresses_in_stack_traces` was accidentally disabled in default `config.xml`. It's removed from the config now, so the setting is enabled by default. [#40749](https://github.com/ClickHouse/ClickHouse/pull/40749) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#40944](https://github.com/ClickHouse/ClickHouse/issues/40944): Fix issue with passing MySQL timeouts for MySQL database engine and MySQL table function. Closes [#34168](https://github.com/ClickHouse/ClickHouse/issues/34168)?notification_referrer_id=NT_kwDOAzsV57MzMDMxNjAzNTY5OjU0MjAzODc5. [#40751](https://github.com/ClickHouse/ClickHouse/pull/40751) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Build/Testing/Packaging Improvement +* Backported in [#41157](https://github.com/ClickHouse/ClickHouse/issues/41157): Add macOS binaries to GH release assets, it fixes [#37718](https://github.com/ClickHouse/ClickHouse/issues/37718). [#41088](https://github.com/ClickHouse/ClickHouse/pull/41088) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#40866](https://github.com/ClickHouse/ClickHouse/issues/40866): - Fix crash while parsing values of type `Object` that contains arrays of variadic dimension. [#40483](https://github.com/ClickHouse/ClickHouse/pull/40483) ([Duc Canh Le](https://github.com/canhld94)). +* Backported in [#40805](https://github.com/ClickHouse/ClickHouse/issues/40805): During insertion of a new query to the `ProcessList` allocations happen. If we reach the memory limit during these allocations we can not use `OvercommitTracker`, because `ProcessList::mutex` is already acquired. Fixes [#40611](https://github.com/ClickHouse/ClickHouse/issues/40611). [#40677](https://github.com/ClickHouse/ClickHouse/pull/40677) ([Dmitry Novik](https://github.com/novikd)). +* Backported in [#40777](https://github.com/ClickHouse/ClickHouse/issues/40777): Fix memory leak while pushing to MVs w/o query context (from Kafka/...). [#40732](https://github.com/ClickHouse/ClickHouse/pull/40732) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#41135](https://github.com/ClickHouse/ClickHouse/issues/41135): Fix access rights for `DESCRIBE TABLE url()` and some other `DESCRIBE TABLE ()`. [#40975](https://github.com/ClickHouse/ClickHouse/pull/40975) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#41242](https://github.com/ClickHouse/ClickHouse/issues/41242): Fixed "possible deadlock avoided" error on automatic conversion of database engine from Ordinary to Atomic. [#41146](https://github.com/ClickHouse/ClickHouse/pull/41146) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#41234](https://github.com/ClickHouse/ClickHouse/issues/41234): Fix background clean up of broken detached parts. [#41190](https://github.com/ClickHouse/ClickHouse/pull/41190) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* use ROBOT_CLICKHOUSE_COMMIT_TOKEN for create-pull-request [#40067](https://github.com/ClickHouse/ClickHouse/pull/40067) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* use input token instead of env var [#40421](https://github.com/ClickHouse/ClickHouse/pull/40421) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* CaresPTRResolver small safety improvement [#40890](https://github.com/ClickHouse/ClickHouse/pull/40890) ([Arthur Passos](https://github.com/arthurpassos)). + diff --git a/docs/en/engines/table-engines/special/materializedview.md b/docs/en/engines/table-engines/special/materializedview.md index 7b06560ec98..807f7bc9eae 100644 --- a/docs/en/engines/table-engines/special/materializedview.md +++ b/docs/en/engines/table-engines/special/materializedview.md @@ -8,4 +8,4 @@ sidebar_label: MaterializedView Used for implementing materialized views (for more information, see [CREATE VIEW](../../../sql-reference/statements/create/view.md#materialized)). For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses that engine. -[Original article](https://clickhouse.com/docs/en/engines/table-engines/special/materializedview/) +[Original article](https://clickhouse.com/docs/en/sql-reference/statements/create/view#materialized-view) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 640c49377d0..36e30360384 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -2,10 +2,9 @@ slug: /en/interfaces/formats sidebar_position: 21 sidebar_label: Input and Output Formats +title: Formats for Input and Output Data --- -# Formats for Input and Output Data - ClickHouse can accept and return data in various formats. A format supported for input can be used to parse the data provided to `INSERT`s, to perform `SELECT`s from a file-backed table such as File, URL or HDFS, or to read an external dictionary. A format supported for output can be used to arrange the results of a `SELECT`, and to perform `INSERT`s into a file-backed table. diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index de496546cb4..aede128e9a4 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -103,6 +103,7 @@ ClickHouse, Inc. does **not** maintain the tools and libraries listed below and - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - [ClickHouse.Net.Migrations](https://github.com/ilyabreev/ClickHouse.Net.Migrations) + - [Linq To DB](https://github.com/linq2db/linq2db) - Elixir - [Ecto](https://github.com/elixir-ecto/ecto) - [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto) diff --git a/docs/en/operations/opentelemetry.md b/docs/en/operations/opentelemetry.md index 8a2e0fbf9ea..db4db18cca5 100644 --- a/docs/en/operations/opentelemetry.md +++ b/docs/en/operations/opentelemetry.md @@ -1,8 +1,8 @@ --- slug: /en/operations/opentelemetry sidebar_position: 62 -sidebar_label: OpenTelemetry Support -title: "[experimental] OpenTelemetry Support" +sidebar_label: Tracing ClickHouse with OpenTelemetry +title: "[experimental] Tracing ClickHouse with OpenTelemetry" --- [OpenTelemetry](https://opentelemetry.io/) is an open standard for collecting traces and metrics from the distributed application. ClickHouse has some support for OpenTelemetry. diff --git a/docs/en/operations/settings/constraints-on-settings.md b/docs/en/operations/settings/constraints-on-settings.md index 4bef197b6cb..651b6465f7e 100644 --- a/docs/en/operations/settings/constraints-on-settings.md +++ b/docs/en/operations/settings/constraints-on-settings.md @@ -26,13 +26,33 @@ The constraints are defined as the following: + + lower_boundary + upper_boundary + + ``` If the user tries to violate the constraints an exception is thrown and the setting isn’t changed. -There are supported three types of constraints: `min`, `max`, `readonly`. The `min` and `max` constraints specify upper and lower boundaries for a numeric setting and can be used in combination. The `readonly` constraint specifies that the user cannot change the corresponding setting at all. +There are supported few types of constraints: `min`, `max`, `readonly` (with alias `const`) and `changeable_in_readonly`. The `min` and `max` constraints specify upper and lower boundaries for a numeric setting and can be used in combination. The `readonly` or `const` constraint specifies that the user cannot change the corresponding setting at all. The `changeable_in_readonly` constraint type allows user to change the setting within `min`/`max` range even if `readonly` setting is set to 1, otherwise settings are not allow to be changed in `readonly=1` mode. Note that `changeable_in_readonly` is supported only if `settings_constraints_replace_previous` is enabled: +``` xml + + true + +``` + +If there are multiple profiles active for a user, then constraints are merged. Merge process depends on `settings_constraints_replace_previous`: +- **true** (recommended): constraints for the same setting are replaced during merge, such that the last constraint is used and all previous are ignored including fields that are not set in new constraint. +- **false** (default): constraints for the same setting are merged in a way that every not set type of constraint is taken from previous profile and every set type of constraint is replaced by value from new profile. + +Read-only mode is enabled by `readonly` setting (not to confuse with `readonly` constraint type): +- `readonly=0`: No read-only restrictions. +- `readonly=1`: Only read queries are allowed and settings cannot be changes unless `changeable_in_readonly` is set. +- `readonly=2`: Only read queries are allowed, but settings can be changed, except for `readonly` setting itself. + **Example:** Let `users.xml` includes lines: diff --git a/docs/en/operations/settings/permissions-for-queries.md b/docs/en/operations/settings/permissions-for-queries.md index c183b159423..3ba62b78cfe 100644 --- a/docs/en/operations/settings/permissions-for-queries.md +++ b/docs/en/operations/settings/permissions-for-queries.md @@ -37,8 +37,7 @@ After setting `readonly = 1`, the user can’t change `readonly` and `allow_ddl` When using the `GET` method in the [HTTP interface](../../interfaces/http.md), `readonly = 1` is set automatically. To modify data, use the `POST` method. -Setting `readonly = 1` prohibit the user from changing all the settings. There is a way to prohibit the user -from changing only specific settings, for details see [constraints on settings](../../operations/settings/constraints-on-settings.md). +Setting `readonly = 1` prohibit the user from changing all the settings. There is a way to prohibit the user from changing only specific settings. Also there is a way to allow changing only specific settings under `readonly = 1` restrictions. For details see [constraints on settings](../../operations/settings/constraints-on-settings.md). Default value: 0 diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 3869168becd..99dc1885354 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3145,6 +3145,17 @@ Result: └─────┴─────┴───────┘ ``` +## enable_extended_results_for_datetime_functions {#enable-extended-results-for-datetime-functions} + +Enables or disables returning results of type `Date32` with extended range (compared to type `Date`) for functions [toStartOfYear](../../sql-reference/functions/date-time-functions.md#tostartofyear), [toStartOfISOYear](../../sql-reference/functions/date-time-functions.md#tostartofisoyear), [toStartOfQuarter](../../sql-reference/functions/date-time-functions.md#tostartofquarter), [toStartOfMonth](../../sql-reference/functions/date-time-functions.md#tostartofmonth), [toStartOfWeek](../../sql-reference/functions/date-time-functions.md#tostartofweek), [toMonday](../../sql-reference/functions/date-time-functions.md#tomonday) and [toLastDayOfMonth](../../sql-reference/functions/date-time-functions.md#tolastdayofmonth). + +Possible values: + +- 0 — Functions return `Date` for all types of arguments. +- 1 — Functions return `Date32` for `Date32` or `DateTime64` arguments and `Date` otherwise. + +Default value: `0`. + ## optimize_move_to_prewhere {#optimize_move_to_prewhere} Enables or disables automatic [PREWHERE](../../sql-reference/statements/select/prewhere.md) optimization in [SELECT](../../sql-reference/statements/select/index.md) queries. @@ -3422,7 +3433,7 @@ Possible values: - 0 — Disabled. - 1 — Enabled. -Default value: 0. +Default value: 1. ## input_format_with_names_use_header {#input_format_with_names_use_header} @@ -3530,8 +3541,8 @@ desc format(JSONEachRow, '{"x" : 1, "y" : "String", "z" : "0.0.0.0" }') settings Result: ```sql -x UInt8 -y Nullable(String) +x UInt8 +y Nullable(String) z IPv4 ``` diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index 546e3d7b7a6..1895e0986b5 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -134,6 +134,13 @@ Example of configuration for versions later or equal to 22.8: 10000000 + + +
+ cache +
+ + ``` @@ -148,9 +155,16 @@ Example of configuration for versions earlier than 22.8: ... ... s3 configuration ... 1 - 10000000 + 10000000 + + +
+ s3 +
+ + ``` @@ -166,7 +180,7 @@ Cache **configuration settings**: - `enable_cache_hits_threshold` - a number, which defines how many times some data needs to be read before it will be cached. Default: `0`, e.g. the data is cached at the first attempt to read it. -- `do_not_evict_index_and_mark_files` - do not evict small frequently used files according to cache policy. Default: `true`. +- `do_not_evict_index_and_mark_files` - do not evict small frequently used files according to cache policy. Default: `false`. This setting was added in version 22.8. If you used filesystem cache before this version, then it will not work on versions starting from 22.8 if this setting is set to `true`. If you want to use this setting, clear old cache created before version 22.8 before upgrading. - `max_file_segment_size` - a maximum size of a single cache file. Default: `104857600` (100 Mb). diff --git a/docs/en/operations/troubleshooting.md b/docs/en/operations/troubleshooting.md index 5a61359a2c0..93bd56087a2 100644 --- a/docs/en/operations/troubleshooting.md +++ b/docs/en/operations/troubleshooting.md @@ -2,10 +2,9 @@ slug: /en/operations/troubleshooting sidebar_position: 46 sidebar_label: Troubleshooting +title: Troubleshooting --- -# Troubleshooting - - [Installation](#troubleshooting-installation-errors) - [Connecting to the server](#troubleshooting-accepts-no-connections) - [Query processing](#troubleshooting-does-not-process-queries) diff --git a/docs/en/sql-reference/data-types/geo.md b/docs/en/sql-reference/data-types/geo.md index 65fd641bec9..48dce40986e 100644 --- a/docs/en/sql-reference/data-types/geo.md +++ b/docs/en/sql-reference/data-types/geo.md @@ -7,13 +7,8 @@ title: "Geo Data Types" ClickHouse supports data types for representing geographical objects — locations, lands, etc. -:::warning -Currently geo data types are an experimental feature. To work with them you must set `allow_experimental_geo_types = 1`. -::: - **See Also** - [Representing simple geographical features](https://en.wikipedia.org/wiki/GeoJSON). -- [allow_experimental_geo_types](../../operations/settings/settings.md#allow-experimental-geo-types) setting. ## Point @@ -24,7 +19,6 @@ Currently geo data types are an experimental feature. To work with them you must Query: ```sql -SET allow_experimental_geo_types = 1; CREATE TABLE geo_point (p Point) ENGINE = Memory(); INSERT INTO geo_point VALUES((10, 10)); SELECT p, toTypeName(p) FROM geo_point; @@ -46,7 +40,6 @@ Result: Query: ```sql -SET allow_experimental_geo_types = 1; CREATE TABLE geo_ring (r Ring) ENGINE = Memory(); INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)]); SELECT r, toTypeName(r) FROM geo_ring; @@ -68,7 +61,6 @@ Result: This is a polygon with one hole: ```sql -SET allow_experimental_geo_types = 1; CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory(); INSERT INTO geo_polygon VALUES([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]]); SELECT pg, toTypeName(pg) FROM geo_polygon; @@ -91,7 +83,6 @@ Result: This multipolygon consists of two separate polygons — the first one without holes, and the second with one hole: ```sql -SET allow_experimental_geo_types = 1; CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory(); INSERT INTO geo_multipolygon VALUES([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]]); SELECT mpg, toTypeName(mpg) FROM geo_multipolygon; diff --git a/docs/en/sql-reference/formats.mdx b/docs/en/sql-reference/formats.mdx new file mode 100644 index 00000000000..013b182091b --- /dev/null +++ b/docs/en/sql-reference/formats.mdx @@ -0,0 +1,10 @@ +--- +slug: /en/sql-reference/formats +sidebar_position: 50 +sidebar_label: Input and Output Formats +title: Formats for Input and Output Data +--- + +import Content from '@site/docs/en/interfaces/formats.md'; + + diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index ced96078ce1..001c7822433 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -268,13 +268,15 @@ Result: ``` :::note -The return type of `toStartOf*`, `toLastDayOfMonth`, `toMonday` functions described below is `Date` or `DateTime`. -Though these functions can take values of the extended types `Date32` and `DateTime64` as an argument, passing them a time outside the normal range (year 1970 to 2149 for `Date` / 2106 for `DateTime`) will produce wrong results. -In case argument is out of normal range: +The return type of `toStartOf*`, `toLastDayOfMonth`, `toMonday` functions described below is determined by the configuration parameter [enable_extended_results_for_datetime_functions](../../operations/settings/settings#enable-extended-results-for-datetime-functions) which is `0` by default. + +Behavior for +* `enable_extended_results_for_datetime_functions = 0`: Functions `toStartOf*`, `toLastDayOfMonth`, `toMonday` return `Date` or `DateTime`. Though these functions can take values of the extended types `Date32` and `DateTime64` as an argument, passing them a time outside the normal range (year 1970 to 2149 for `Date` / 2106 for `DateTime`) will produce wrong results. In case argument is out of normal range: * If the argument is smaller than 1970, the result will be calculated from the argument `1970-01-01 (00:00:00)` instead. * If the return type is `DateTime` and the argument is larger than `2106-02-07 08:28:15`, the result will be calculated from the argument `2106-02-07 08:28:15` instead. * If the return type is `Date` and the argument is larger than `2149-06-06`, the result will be calculated from the argument `2149-06-06` instead. * If `toLastDayOfMonth` is called with an argument greater then `2149-05-31`, the result will be calculated from the argument `2149-05-31` instead. +* `enable_extended_results_for_datetime_functions = 1`: Functions `toStartOfYear`, `toStartOfISOYear`, `toStartOfQuarter`, `toStartOfMonth`, `toStartOfWeek`, `toLastDayOfMonth`, `toMonday` return `Date` or `DateTime` if their argument is a `Date` or `DateTime`, and they return `Date32` or `DateTime64` if their argument is a `Date32` or `DateTime64`. ::: ## toStartOfYear @@ -303,6 +305,8 @@ Returns the date. Rounds up a date or date with time to the last day of the month. Returns the date. +If `toLastDayOfMonth` is called with an argument of type `Date` greater then 2149-05-31, the result will be calculated from the argument 2149-05-31 instead. + ## toMonday Rounds down a date or date with time to the nearest Monday. @@ -640,7 +644,7 @@ Result: ## date\_diff -Returns the difference between two dates or dates with time values. +Returns the difference between two dates or dates with time values. The difference is calculated using relative units, e.g. the difference between `2022-01-01` and `2021-12-29` is 3 days for day unit (see [toRelativeDayNum](#torelativedaynum)), 1 month for month unit (see [toRelativeMonthNum](#torelativemonthnum)), 1 year for year unit (see [toRelativeYearNum](#torelativeyearnum)). **Syntax** @@ -1059,9 +1063,9 @@ SELECT ## timeSlots(StartTime, Duration,\[, Size\]) -For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. -This is necessary, for example, when searching for pageviews in the corresponding session. -Accepts DateTime and DateTime64 as ’StartTime’ argument. For DateTime, ’Duration’ and ’Size’ arguments must be `UInt32`. For ’DateTime64’ they must be `Decimal64`. +For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. +This is necessary, for example, when searching for pageviews in the corresponding session. +Accepts DateTime and DateTime64 as ’StartTime’ argument. For DateTime, ’Duration’ and ’Size’ arguments must be `UInt32`. For ’DateTime64’ they must be `Decimal64`. Returns an array of DateTime/DateTime64 (return type matches the type of ’StartTime’). For DateTime64, the return value's scale can differ from the scale of ’StartTime’ --- the highest scale among all given arguments is taken. Example: @@ -1227,6 +1231,8 @@ Result: Function converts Unix timestamp to a calendar date and a time of a day. When there is only a single argument of [Integer](../../sql-reference/data-types/int-uint.md) type, it acts in the same way as [toDateTime](../../sql-reference/functions/type-conversion-functions.md#todatetime) and return [DateTime](../../sql-reference/data-types/datetime.md) type. +Alias: `fromUnixTimestamp`. + **Example:** Query: diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index d86eb6b45ae..877179a66a6 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -1823,6 +1823,36 @@ Result: Evaluate external model. Accepts a model name and model arguments. Returns Float64. +## catboostEvaluate(path_to_model, feature_1, feature_2, …, feature_n) + +Evaluate external catboost model. [CatBoost](https://catboost.ai) is an open-source gradient boosting library developed by Yandex for machine learing. +Accepts a path to a catboost model and model arguments (features). Returns Float64. + +``` sql +SELECT feat1, ..., feat_n, catboostEvaluate('/path/to/model.bin', feat_1, ..., feat_n) AS prediction +FROM data_table +``` + +**Prerequisites** + +1. Build the catboost evaluation library + +Before evaluating catboost models, the `libcatboostmodel.` library must be made available. See [CatBoost documentation](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html) how to compile it. + +Next, specify the path to `libcatboostmodel.` in the clickhouse configuration: + +``` xml + +... + /path/to/libcatboostmodel.so +... + +``` + +2. Train a catboost model using libcatboost + +See [Training and applying models](https://catboost.ai/docs/features/training.html#training) for how to train catboost models from a training data set. + ## throwIf(x\[, message\[, error_code\]\]) Throw an exception if the argument is non zero. diff --git a/docs/en/sql-reference/statements/alter/role.md b/docs/en/sql-reference/statements/alter/role.md index 2bee9fd0dc6..c068d6c4fce 100644 --- a/docs/en/sql-reference/statements/alter/role.md +++ b/docs/en/sql-reference/statements/alter/role.md @@ -13,5 +13,5 @@ Syntax: ``` sql ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` diff --git a/docs/en/sql-reference/statements/alter/settings-profile.md b/docs/en/sql-reference/statements/alter/settings-profile.md index 234bb22ae14..adfe65e3c80 100644 --- a/docs/en/sql-reference/statements/alter/settings-profile.md +++ b/docs/en/sql-reference/statements/alter/settings-profile.md @@ -13,5 +13,5 @@ Syntax: ``` sql ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` diff --git a/docs/en/sql-reference/statements/create/role.md b/docs/en/sql-reference/statements/create/role.md index 84b02042fe9..6c80204688b 100644 --- a/docs/en/sql-reference/statements/create/role.md +++ b/docs/en/sql-reference/statements/create/role.md @@ -11,7 +11,7 @@ Syntax: ``` sql CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [, name2 ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` ## Managing Roles diff --git a/docs/en/sql-reference/statements/create/settings-profile.md b/docs/en/sql-reference/statements/create/settings-profile.md index 4c7e4c30ea0..8883b22896b 100644 --- a/docs/en/sql-reference/statements/create/settings-profile.md +++ b/docs/en/sql-reference/statements/create/settings-profile.md @@ -12,7 +12,7 @@ Syntax: ``` sql CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` `ON CLUSTER` clause allows creating settings profiles on a cluster, see [Distributed DDL](../../../sql-reference/distributed-ddl.md). diff --git a/docs/en/sql-reference/statements/delete.md b/docs/en/sql-reference/statements/delete.md index 487dfc87f9a..0dc6cc0d09a 100644 --- a/docs/en/sql-reference/statements/delete.md +++ b/docs/en/sql-reference/statements/delete.md @@ -32,6 +32,12 @@ SET allow_experimental_lightweight_delete = true; An [alternative way to delete rows](./alter/delete.md) in ClickHouse is `ALTER TABLE ... DELETE`, which might be more efficient if you do bulk deletes only occasionally and don't need the operation to be applied instantly. In most use cases the new lightweight `DELETE FROM` behavior will be considerably faster. :::warning -Even though deletes are becoming more lightweight in ClickHouse, they should still not be used as aggressively as on OLTP system. Ligthweight deletes are currently efficient for wide parts, but for compact parts they can be a heavyweight operation, and it may be better to use `ALTER TABLE` for some scenarios. +Even though deletes are becoming more lightweight in ClickHouse, they should still not be used as aggressively as on an OLTP system. Ligthweight deletes are currently efficient for wide parts, but for compact parts they can be a heavyweight operation, and it may be better to use `ALTER TABLE` for some scenarios. ::: +:::note +`DELETE FROM` requires the `ALTER DELETE` privilege: +```sql +grant ALTER DELETE ON db.table to username; +``` +::: diff --git a/docs/en/sql-reference/statements/select/group-by.md b/docs/en/sql-reference/statements/select/group-by.md index b5e194343ca..ac02e9ab5a1 100644 --- a/docs/en/sql-reference/statements/select/group-by.md +++ b/docs/en/sql-reference/statements/select/group-by.md @@ -213,9 +213,10 @@ If the `WITH TOTALS` modifier is specified, another row will be calculated. This This extra row is only produced in `JSON*`, `TabSeparated*`, and `Pretty*` formats, separately from the other rows: -- In `JSON*` formats, this row is output as a separate ‘totals’ field. -- In `TabSeparated*` formats, the row comes after the main result, preceded by an empty row (after the other data). +- In `XML` and `JSON*` formats, this row is output as a separate ‘totals’ field. +- In `TabSeparated*`, `CSV*` and `Vertical` formats, the row comes after the main result, preceded by an empty row (after the other data). - In `Pretty*` formats, the row is output as a separate table after the main result. +- In `Template` format, the row is output according to specified template. - In the other formats it is not available. :::note diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index c1692bd9b29..de190aaf982 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -135,9 +135,9 @@ In all other cases, we do not recommend using the asterisk, since it only gives In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the **extremes** setting to 1. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output. -An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats. +An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `XML`, `JSON*`, `TabSeparated*`, `CSV*`, `Vertical`, `Template` and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats. -In `JSON*` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. +In `JSON*` and `XML` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*`, `CSV*` and `Vertical` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. In `Template` format the extreme values are output according to specified template. Extreme values are calculated for rows before `LIMIT`, but after `LIMIT BY`. However, when using `LIMIT offset, size`, the rows before `offset` are included in `extremes`. In stream requests, the result may also include a small number of rows that passed through `LIMIT`. diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 9b7527caaa9..feeefd5502a 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -6,45 +6,6 @@ sidebar_label: SYSTEM # SYSTEM Statements -The list of available `SYSTEM` statements: - -- [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries) -- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) -- [RELOAD DICTIONARY](#query_language-system-reload-dictionary) -- [RELOAD MODELS](#query_language-system-reload-models) -- [RELOAD MODEL](#query_language-system-reload-model) -- [RELOAD FUNCTIONS](#query_language-system-reload-functions) -- [RELOAD FUNCTION](#query_language-system-reload-functions) -- [DROP DNS CACHE](#query_language-system-drop-dns-cache) -- [DROP MARK CACHE](#query_language-system-drop-mark-cache) -- [DROP UNCOMPRESSED CACHE](#query_language-system-drop-uncompressed-cache) -- [DROP COMPILED EXPRESSION CACHE](#query_language-system-drop-compiled-expression-cache) -- [DROP REPLICA](#query_language-system-drop-replica) -- [FLUSH LOGS](#query_language-system-flush_logs) -- [RELOAD CONFIG](#query_language-system-reload-config) -- [SHUTDOWN](#query_language-system-shutdown) -- [KILL](#query_language-system-kill) -- [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) -- [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) -- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) -- [STOP MERGES](#query_language-system-stop-merges) -- [START MERGES](#query_language-system-start-merges) -- [STOP TTL MERGES](#query_language-stop-ttl-merges) -- [START TTL MERGES](#query_language-start-ttl-merges) -- [STOP MOVES](#query_language-stop-moves) -- [START MOVES](#query_language-start-moves) -- [SYSTEM UNFREEZE](#query_language-system-unfreeze) -- [STOP FETCHES](#query_language-system-stop-fetches) -- [START FETCHES](#query_language-system-start-fetches) -- [STOP REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [START REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [STOP REPLICATION QUEUES](#query_language-system-stop-replication-queues) -- [START REPLICATION QUEUES](#query_language-system-start-replication-queues) -- [SYNC REPLICA](#query_language-system-sync-replica) -- [RESTART REPLICA](#query_language-system-restart-replica) -- [RESTORE REPLICA](#query_language-system-restore-replica) -- [RESTART REPLICAS](#query_language-system-restart-replicas) - ## RELOAD EMBEDDED DICTIONARIES Reload all [Internal dictionaries](../../sql-reference/dictionaries/internal-dicts.md). @@ -69,7 +30,12 @@ SELECT name, status FROM system.dictionaries; ## RELOAD MODELS -Reloads all [CatBoost](../../guides/developer/apply-catboost-model.md) models if the configuration was updated without restarting the server. +:::note +This statement and `SYSTEM RELOAD MODEL` merely unload catboost models from the clickhouse-library-bridge. The function `catboostEvaluate()` +loads a model upon first access if it is not loaded yet. +::: + +Unloads all CatBoost models. **Syntax** @@ -79,12 +45,12 @@ SYSTEM RELOAD MODELS [ON CLUSTER cluster_name] ## RELOAD MODEL -Completely reloads a CatBoost model `model_name` if the configuration was updated without restarting the server. +Unloads a CatBoost model at `model_path`. **Syntax** ```sql -SYSTEM RELOAD MODEL [ON CLUSTER cluster_name] +SYSTEM RELOAD MODEL [ON CLUSTER cluster_name] ``` ## RELOAD FUNCTIONS diff --git a/docs/en/sql-reference/table-functions/file.md b/docs/en/sql-reference/table-functions/file.md index a110bfbd15c..f40107aaaca 100644 --- a/docs/en/sql-reference/table-functions/file.md +++ b/docs/en/sql-reference/table-functions/file.md @@ -13,7 +13,7 @@ Creates a table from a file. This table function is similar to [url](../../sql-r **Syntax** ``` sql -file(path, format, structure) +file(path [,format] [,structure]) ``` **Parameters** diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index 2df7d6e46b3..545037665bb 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -11,7 +11,7 @@ Provides table-like interface to select/insert files in [Amazon S3](https://aws. **Syntax** ``` sql -s3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, [compression]) +s3(path [,aws_access_key_id, aws_secret_access_key] [,format] [,structure] [,compression]) ``` **Arguments** diff --git a/docs/en/sql-reference/table-functions/s3Cluster.md b/docs/en/sql-reference/table-functions/s3Cluster.md index 9d006af9572..b81fc51fd18 100644 --- a/docs/en/sql-reference/table-functions/s3Cluster.md +++ b/docs/en/sql-reference/table-functions/s3Cluster.md @@ -10,7 +10,7 @@ Allows processing files from [Amazon S3](https://aws.amazon.com/s3/) in parallel **Syntax** ``` sql -s3Cluster(cluster_name, source, [access_key_id, secret_access_key,] format, structure) +s3Cluster(cluster_name, source, [,access_key_id, secret_access_key] [,format] [,structure]) ``` **Arguments** diff --git a/docs/en/sql-reference/table-functions/url.md b/docs/en/sql-reference/table-functions/url.md index f1ed7b4dfe4..014dc3ae853 100644 --- a/docs/en/sql-reference/table-functions/url.md +++ b/docs/en/sql-reference/table-functions/url.md @@ -13,7 +13,7 @@ sidebar_label: url **Syntax** ``` sql -url(URL, format, structure) +url(URL [,format] [,structure]) ``` **Parameters** diff --git a/docs/redirects.txt b/docs/redirects.txt index 949b9d48ca8..cea138f7237 100644 --- a/docs/redirects.txt +++ b/docs/redirects.txt @@ -155,7 +155,6 @@ getting_started/index.md getting-started/index.md getting_started/install.md getting-started/install.md getting_started/playground.md getting-started/playground.md getting_started/tutorial.md getting-started/tutorial.md -guides/apply_catboost_model.md guides/apply-catboost-model.md images/column_oriented.gif images/column-oriented.gif images/row_oriented.gif images/row-oriented.gif interfaces/http_interface.md interfaces/http.md diff --git a/docs/ru/guides/apply-catboost-model.md b/docs/ru/guides/apply-catboost-model.md deleted file mode 100644 index 68d7042df2d..00000000000 --- a/docs/ru/guides/apply-catboost-model.md +++ /dev/null @@ -1,241 +0,0 @@ ---- -slug: /ru/guides/apply-catboost-model -sidebar_position: 41 -sidebar_label: "Применение модели CatBoost в ClickHouse" ---- - -# Применение модели CatBoost в ClickHouse {#applying-catboost-model-in-clickhouse} - -[CatBoost](https://catboost.ai) — открытая программная библиотека разработанная компанией [Яндекс](https://yandex.ru/company/) для машинного обучения, которая использует схему градиентного бустинга. - -С помощью этой инструкции вы научитесь применять предобученные модели в ClickHouse: в результате вы запустите вывод модели из SQL. - -Чтобы применить модель CatBoost в ClickHouse: - -1. [Создайте таблицу](#create-table). -2. [Вставьте данные в таблицу](#insert-data-to-table). -3. [Интегрируйте CatBoost в ClickHouse](#integrate-catboost-into-clickhouse) (Опциональный шаг). -4. [Запустите вывод модели из SQL](#run-model-inference). - -Подробнее об обучении моделей в CatBoost, см. [Обучение и применение моделей](https://catboost.ai/docs/features/training.html#training). - -Вы можете перегрузить модели CatBoost, если их конфигурация была обновлена, без перезагрузки сервера. Для этого используйте системные запросы [RELOAD MODEL](../sql-reference/statements/system.md#query_language-system-reload-model) и [RELOAD MODELS](../sql-reference/statements/system.md#query_language-system-reload-models). - -## Перед началом работы {#prerequisites} - -Если у вас еще нет [Docker](https://docs.docker.com/install/), установите его. - - :::note "Примечание" - [Docker](https://www.docker.com) – это программная платформа для создания контейнеров, которые изолируют установку CatBoost и ClickHouse от остальной части системы. - ::: -Перед применением модели CatBoost: - -**1.** Скачайте [Docker-образ](https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) из реестра: - -``` bash -$ docker pull yandex/tutorial-catboost-clickhouse -``` - -Данный Docker-образ содержит все необходимое для запуска CatBoost и ClickHouse: код, среду выполнения, библиотеки, переменные окружения и файлы конфигурации. - -**2.** Проверьте, что Docker-образ успешно скачался: - -``` bash -$ docker image ls -REPOSITORY TAG IMAGE ID CREATED SIZE -yandex/tutorial-catboost-clickhouse latest 622e4d17945b 22 hours ago 1.37GB -``` - -**3.** Запустите Docker-контейнер основанный на данном образе: - -``` bash -$ docker run -it -p 8888:8888 yandex/tutorial-catboost-clickhouse -``` - -## 1. Создайте таблицу {#create-table} - -Чтобы создать таблицу для обучающей выборки: - -**1.** Запустите клиент ClickHouse: - -``` bash -$ clickhouse client -``` - - :::note "Примечание" - Сервер ClickHouse уже запущен внутри Docker-контейнера. - ::: -**2.** Создайте таблицу в ClickHouse с помощью следующей команды: - -``` sql -:) CREATE TABLE amazon_train -( - date Date MATERIALIZED today(), - ACTION UInt8, - RESOURCE UInt32, - MGR_ID UInt32, - ROLE_ROLLUP_1 UInt32, - ROLE_ROLLUP_2 UInt32, - ROLE_DEPTNAME UInt32, - ROLE_TITLE UInt32, - ROLE_FAMILY_DESC UInt32, - ROLE_FAMILY UInt32, - ROLE_CODE UInt32 -) -ENGINE = MergeTree ORDER BY date -``` - -**3.** Выйдите из клиента ClickHouse: - -``` sql -:) exit -``` - -## 2. Вставьте данные в таблицу {#insert-data-to-table} - -Чтобы вставить данные: - -**1.** Выполните следующую команду: - -``` bash -$ clickhouse client --host 127.0.0.1 --query 'INSERT INTO amazon_train FORMAT CSVWithNames' < ~/amazon/train.csv -``` - -**2.** Запустите клиент ClickHouse: - -``` bash -$ clickhouse client -``` - -**3.** Проверьте, что данные успешно загрузились: - -``` sql -:) SELECT count() FROM amazon_train - -SELECT count() -FROM amazon_train - -+-count()-+ -| 65538 | -+---------+ -``` - -## 3. Интегрируйте CatBoost в ClickHouse {#integrate-catboost-into-clickhouse} - - :::note "Примечание" - **Опциональный шаг.** Docker-образ содержит все необходимое для запуска CatBoost и ClickHouse. - ::: -Чтобы интегрировать CatBoost в ClickHouse: - -**1.** Создайте библиотеку для оценки модели. - -Наиболее быстрый способ оценить модель CatBoost — это скомпилировать библиотеку `libcatboostmodel.`. Подробнее о том, как скомпилировать библиотеку, читайте в [документации CatBoost](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). - -**2.** Создайте в любом месте новую директорию с произвольным названием, например `data` и поместите в нее созданную библиотеку. Docker-образ уже содержит библиотеку `data/libcatboostmodel.so`. - -**3.** Создайте в любом месте новую директорию для конфигурации модели с произвольным названием, например `models`. - -**4.** Создайте файл конфигурации модели с произвольным названием, например `models/amazon_model.xml`. - -**5.** Опишите конфигурацию модели: - -``` xml - - - - catboost - - amazon - - /home/catboost/tutorial/catboost_model.bin - - 0 - - -``` - -**6.** Добавьте в конфигурацию ClickHouse путь к CatBoost и конфигурации модели: - -``` xml - -/home/catboost/data/libcatboostmodel.so -/home/catboost/models/*_model.xml -``` - :::note "Примечание" - Вы можете позднее изменить путь к конфигурации модели CatBoost без перезагрузки сервера. - ::: -## 4. Запустите вывод модели из SQL {#run-model-inference} - -Для тестирования модели запустите клиент ClickHouse `$ clickhouse client`. - -Проверьте, что модель работает: - -``` sql -:) SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) > 0 AS prediction, - ACTION AS target -FROM amazon_train -LIMIT 10 -``` - - :::note "Примечание" - Функция [modelEvaluate](../sql-reference/functions/other-functions.md#function-modelevaluate) возвращает кортежи (tuple) с исходными прогнозами по классам для моделей с несколькими классами. - ::: -Спрогнозируйте вероятность: - -``` sql -:) SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) AS prediction, - 1. / (1 + exp(-prediction)) AS probability, - ACTION AS target -FROM amazon_train -LIMIT 10 -``` - - :::note "Примечание" - Подробнее про функцию [exp()](../sql-reference/functions/math-functions.md). - ::: -Посчитайте логистическую функцию потерь (LogLoss) на всей выборке: - -``` sql -:) SELECT -avg(tg * log(prob) + (1 - tg) * log(1 - prob)) AS logloss -FROM -( - SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) AS prediction, - 1. / (1. + exp(-prediction)) AS prob, - ACTION AS tg - FROM amazon_train -) -``` - - :::note "Примечание" - Подробнее про функции [avg()](../sql-reference/aggregate-functions/reference/avg.md#agg_function-avg), [log()](../sql-reference/functions/math-functions.md). - ::: \ No newline at end of file diff --git a/docs/ru/guides/index.md b/docs/ru/guides/index.md index 0b5938dfc09..882f71b5700 100644 --- a/docs/ru/guides/index.md +++ b/docs/ru/guides/index.md @@ -7,5 +7,3 @@ sidebar_label: "Руководства" # Руководства {#rukovodstva} Подробные пошаговые инструкции, которые помогут вам решать различные задачи с помощью ClickHouse. - -- [Применение модели CatBoost в ClickHouse](apply-catboost-model.md) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 5ddc684ce2a..0d4f0c63210 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -3799,6 +3799,17 @@ Exception: Total regexp lengths too large. Значение по умолчанию: `1`. +## enable_extended_results_for_datetime_functions {#enable-extended-results-for-datetime-functions} + +Включает или отключает возвращение результатов типа `Date32` с расширенным диапазоном (по сравнению с типом `Date`) для функций [toStartOfYear](../../sql-reference/functions/date-time-functions.md#tostartofyear), [toStartOfISOYear](../../sql-reference/functions/date-time-functions.md#tostartofisoyear), [toStartOfQuarter](../../sql-reference/functions/date-time-functions.md#tostartofquarter), [toStartOfMonth](../../sql-reference/functions/date-time-functions.md#tostartofmonth), [toStartOfWeek](../../sql-reference/functions/date-time-functions.md#tostartofweek), [toMonday](../../sql-reference/functions/date-time-functions.md#tomonday) и [toLastDayOfMonth](../../sql-reference/functions/date-time-functions.md#tolastdayofmonth). + +Возможные значения: + +- 0 — Функции возвращают результаты типа `Date` для всех типов аргументов. +- 1 — Функции возвращают результаты типа `Date32` для аргументов типа `Date32` или `DateTime64` и возвращают `Date` в других случаях. + +Значение по умолчанию: `0`. + **Пример** Запрос: diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index 1c623cd1dab..27689426cbe 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -268,24 +268,18 @@ SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp; ``` :::note -Тип возвращаемого описанными далее функциями `toStartOf*`, `toMonday` значения - `Date` или `DateTime`. -Хотя эти функции могут принимать значения типа `Date32` или `DateTime64` в качестве аргумента, при обработке аргумента вне нормального диапазона значений (`1970` - `2148` для `Date` и `1970-01-01 00:00:00`-`2106-02-07 08:28:15` для `DateTime`) будет получен некорректный результат. -Возвращаемые значения для значений вне нормального диапазона: -* `1970-01-01 (00:00:00)` будет возвращён для моментов времени до 1970 года, -* `2106-02-07 08:28:15` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `DateTime`, -* `2149-06-06` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `Date`, -* `2149-05-31` будет результатом функции `toLastDayOfMonth` при обработке аргумента больше `2149-05-31`. +Тип возвращаемого значения описанными далее функциями `toStartOf*`, `toLastDayOfMonth`, `toMonday` определяется конфигурационным параметром [enable_extended_results_for_datetime_functions](../../operations/settings/settings#enable-extended-results-for-datetime-functions) имеющим по умолчанию значение `0`. + +Поведение для +* `enable_extended_results_for_datetime_functions = 0`: Функции `toStartOf*`, `toLastDayOfMonth`, `toMonday` возвращают `Date` или `DateTime`. Хотя эти функции могут принимать значения типа `Date32` или `DateTime64` в качестве аргумента, при обработке аргумента вне нормального диапазона значений (`1970` - `2148` для `Date` и `1970-01-01 00:00:00`-`2106-02-07 08:28:15` для `DateTime`) будет получен некорректный результат. +В случае если значение аргумента вне нормального диапазона: + * `1970-01-01 (00:00:00)` будет возвращён для моментов времени до 1970 года, + * `2106-02-07 08:28:15` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `DateTime`, + * `2149-06-06` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `Date`, + * `2149-05-31` будет результатом функции `toLastDayOfMonth` при обработке аргумента больше `2149-05-31`. +* `enable_extended_results_for_datetime_functions = 1`: Функции `toStartOfYear`, `toStartOfISOYear`, `toStartOfQuarter`, `toStartOfMonth`, `toStartOfWeek`, `toLastDayOfMonth`, `toMonday` возвращают `Date` или `DateTime` если их аргумент `Date` или `DateTime` и они возвращают `Date32` или `DateTime64` если их аргумент `Date32` или `DateTime64`. ::: -:::note -Тип возвращаемого описанными далее функциями `toStartOf*`, `toLastDayOfMonth`, `toMonday` значения - `Date` или `DateTime`. -Хотя эти функции могут принимать значения типа `Date32` или `DateTime64` в качестве аргумента, при обработке аргумента вне нормального диапазона значений (`1970` - `2148` для `Date` и `1970-01-01 00:00:00`-`2106-02-07 08:28:15` для `DateTime`) будет получен некорректный результат. -Возвращаемые значения для значений вне нормального диапазона: -* `1970-01-01 (00:00:00)` будет возвращён для моментов времени до 1970 года, -* `2106-02-07 08:28:15` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `DateTime`, -* `2149-06-06` будет взят в качестве аргумента, если полученный аргумент превосходит данное значение и возвращаемый тип - `Date`. - ::: -* ## toStartOfYear {#tostartofyear} Округляет дату или дату-с-временем вниз до первого дня года. @@ -324,6 +318,8 @@ SELECT toStartOfISOYear(toDate('2017-01-01')) AS ISOYear20170101; Округляет дату или дату-с-временем до последнего числа месяца. Возвращается дата. +Если `toLastDayOfMonth` вызывается с аргументом типа `Date` большим чем 2149-05-31, то результат будет вычислен от аргумента 2149-05-31. + ## toMonday {#tomonday} Округляет дату или дату-с-временем вниз до ближайшего понедельника. @@ -977,7 +973,7 @@ SELECT now('Europe/Moscow'); ## timeSlots(StartTime, Duration,\[, Size\]) {#timeslotsstarttime-duration-size} Для интервала, начинающегося в `StartTime` и длящегося `Duration` секунд, возвращает массив моментов времени, кратных `Size`. Параметр `Size` указывать необязательно, по умолчанию он равен 1800 секундам (30 минутам) - необязательный параметр. Данная функция может использоваться, например, для анализа количества просмотров страницы за соответствующую сессию. -Аргумент `StartTime` может иметь тип `DateTime` или `DateTime64`. В случае, если используется `DateTime`, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`. +Аргумент `StartTime` может иметь тип `DateTime` или `DateTime64`. В случае, если используется `DateTime`, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`. Возвращает массив DateTime/DateTime64 (тип будет совпадать с типом параметра ’StartTime’). Для DateTime64 масштаб(scale) возвращаемой величины может отличаться от масштаба фргумента ’StartTime’ --- результат будет иметь наибольший масштаб среди всех данных аргументов. Пример использования: diff --git a/docs/ru/sql-reference/statements/alter/role.md b/docs/ru/sql-reference/statements/alter/role.md index a86ff780b8d..4e84260fd40 100644 --- a/docs/ru/sql-reference/statements/alter/role.md +++ b/docs/ru/sql-reference/statements/alter/role.md @@ -13,6 +13,6 @@ sidebar_label: ROLE ``` sql ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` diff --git a/docs/ru/sql-reference/statements/alter/settings-profile.md b/docs/ru/sql-reference/statements/alter/settings-profile.md index ec1cd1f72e6..8166f17597c 100644 --- a/docs/ru/sql-reference/statements/alter/settings-profile.md +++ b/docs/ru/sql-reference/statements/alter/settings-profile.md @@ -13,6 +13,6 @@ sidebar_label: SETTINGS PROFILE ``` sql ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` diff --git a/docs/ru/sql-reference/statements/create/role.md b/docs/ru/sql-reference/statements/create/role.md index 9e06ad1914e..4a93de8a74c 100644 --- a/docs/ru/sql-reference/statements/create/role.md +++ b/docs/ru/sql-reference/statements/create/role.md @@ -12,7 +12,7 @@ sidebar_label: "Роль" ```sql CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [, name2 ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` ## Управление ролями {#managing-roles} diff --git a/docs/ru/sql-reference/statements/create/settings-profile.md b/docs/ru/sql-reference/statements/create/settings-profile.md index d85b2aadeda..9aa77e4c241 100644 --- a/docs/ru/sql-reference/statements/create/settings-profile.md +++ b/docs/ru/sql-reference/statements/create/settings-profile.md @@ -13,7 +13,7 @@ sidebar_label: "Профиль настроек" ``` sql CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` Секция `ON CLUSTER` позволяет создавать профили на кластере, см. [Распределенные DDL запросы](../../../sql-reference/distributed-ddl.md). diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index c1dc03a63d1..a7dec7abe27 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -6,43 +6,6 @@ sidebar_label: SYSTEM # Запросы SYSTEM {#query-language-system} -- [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries) -- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) -- [RELOAD DICTIONARY](#query_language-system-reload-dictionary) -- [RELOAD MODELS](#query_language-system-reload-models) -- [RELOAD MODEL](#query_language-system-reload-model) -- [RELOAD FUNCTIONS](#query_language-system-reload-functions) -- [RELOAD FUNCTION](#query_language-system-reload-functions) -- [DROP DNS CACHE](#query_language-system-drop-dns-cache) -- [DROP MARK CACHE](#query_language-system-drop-mark-cache) -- [DROP UNCOMPRESSED CACHE](#query_language-system-drop-uncompressed-cache) -- [DROP COMPILED EXPRESSION CACHE](#query_language-system-drop-compiled-expression-cache) -- [DROP REPLICA](#query_language-system-drop-replica) -- [FLUSH LOGS](#query_language-system-flush_logs) -- [RELOAD CONFIG](#query_language-system-reload-config) -- [SHUTDOWN](#query_language-system-shutdown) -- [KILL](#query_language-system-kill) -- [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) -- [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) -- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) -- [STOP MERGES](#query_language-system-stop-merges) -- [START MERGES](#query_language-system-start-merges) -- [STOP TTL MERGES](#query_language-stop-ttl-merges) -- [START TTL MERGES](#query_language-start-ttl-merges) -- [STOP MOVES](#query_language-stop-moves) -- [START MOVES](#query_language-start-moves) -- [SYSTEM UNFREEZE](#query_language-system-unfreeze) -- [STOP FETCHES](#query_language-system-stop-fetches) -- [START FETCHES](#query_language-system-start-fetches) -- [STOP REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [START REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [STOP REPLICATION QUEUES](#query_language-system-stop-replication-queues) -- [START REPLICATION QUEUES](#query_language-system-start-replication-queues) -- [SYNC REPLICA](#query_language-system-sync-replica) -- [RESTART REPLICA](#query_language-system-restart-replica) -- [RESTORE REPLICA](#query_language-system-restore-replica) -- [RESTART REPLICAS](#query_language-system-restart-replicas) - ## RELOAD EMBEDDED DICTIONARIES] {#query_language-system-reload-emdedded-dictionaries} Перегружает все [Встроенные словари](../dictionaries/internal-dicts.md). По умолчанию встроенные словари выключены. @@ -66,7 +29,12 @@ SELECT name, status FROM system.dictionaries; ## RELOAD MODELS {#query_language-system-reload-models} -Перегружает все модели [CatBoost](../../guides/apply-catboost-model.md#applying-catboost-model-in-clickhouse), если их конфигурация была обновлена, без перезагрузки сервера. +:::note +Это утверждение и `SYSTEM RELOAD MODEL` просто выгружают модели catboost из clickhouse-library-bridge. Функция `catboostEvaluate()` +загружает модель при первом обращении, если она еще не загружена. +::: + +Разгрузите все модели CatBoost. **Синтаксис** @@ -76,12 +44,12 @@ SYSTEM RELOAD MODELS ## RELOAD MODEL {#query_language-system-reload-model} -Полностью перегружает модель [CatBoost](../../guides/apply-catboost-model.md#applying-catboost-model-in-clickhouse) `model_name`, если ее конфигурация была обновлена, без перезагрузки сервера. +Выгружает модель CatBoost по адресу `модель_путь`. **Синтаксис** ```sql -SYSTEM RELOAD MODEL +SYSTEM RELOAD MODEL ``` ## RELOAD FUNCTIONS {#query_language-system-reload-functions} diff --git a/docs/ru/sql-reference/table-functions/file.md b/docs/ru/sql-reference/table-functions/file.md index 1f262c9403a..df35a1c4ac0 100644 --- a/docs/ru/sql-reference/table-functions/file.md +++ b/docs/ru/sql-reference/table-functions/file.md @@ -13,7 +13,7 @@ sidebar_label: file **Синтаксис** ``` sql -file(path, format, structure) +file(path [,format] [,structure]) ``` **Параметры** diff --git a/docs/ru/sql-reference/table-functions/s3.md b/docs/ru/sql-reference/table-functions/s3.md index ae0419a4b84..14c8204fd1d 100644 --- a/docs/ru/sql-reference/table-functions/s3.md +++ b/docs/ru/sql-reference/table-functions/s3.md @@ -11,7 +11,7 @@ sidebar_label: s3 **Синтаксис** ``` sql -s3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, [compression]) +s3(path [,aws_access_key_id, aws_secret_access_key] [,format] [,structure] [,compression]) ``` **Aргументы** diff --git a/docs/ru/sql-reference/table-functions/s3Cluster.md b/docs/ru/sql-reference/table-functions/s3Cluster.md index e6b317253c0..1c12913fabe 100644 --- a/docs/ru/sql-reference/table-functions/s3Cluster.md +++ b/docs/ru/sql-reference/table-functions/s3Cluster.md @@ -11,7 +11,7 @@ sidebar_label: s3Cluster **Синтаксис** ``` sql -s3Cluster(cluster_name, source, [access_key_id, secret_access_key,] format, structure) +s3Cluster(cluster_name, source, [,access_key_id, secret_access_key] [,format] [,structure]) ``` **Аргументы** diff --git a/docs/ru/sql-reference/table-functions/url.md b/docs/ru/sql-reference/table-functions/url.md index d4fb11b0de7..e5d9faeec00 100644 --- a/docs/ru/sql-reference/table-functions/url.md +++ b/docs/ru/sql-reference/table-functions/url.md @@ -13,7 +13,7 @@ sidebar_label: url **Синтаксис** ``` sql -url(URL, format, structure) +url(URL [,format] [,structure]) ``` **Параметры** diff --git a/docs/zh/guides/apply-catboost-model.md b/docs/zh/guides/apply-catboost-model.md deleted file mode 100644 index 861e5372875..00000000000 --- a/docs/zh/guides/apply-catboost-model.md +++ /dev/null @@ -1,244 +0,0 @@ ---- -slug: /zh/guides/apply-catboost-model -sidebar_position: 41 -sidebar_label: "\u5E94\u7528CatBoost\u6A21\u578B" ---- - -# 在ClickHouse中应用Catboost模型 {#applying-catboost-model-in-clickhouse} - -[CatBoost](https://catboost.ai) 是一个由[Yandex](https://yandex.com/company/)开发的开源免费机器学习库。 - - -通过本篇文档，您将学会如何用SQL语句调用已经存放在Clickhouse中的预训练模型来预测数据。 - - -为了在ClickHouse中应用CatBoost模型，需要进行如下步骤： - -1. [创建数据表](#create-table). -2. [将数据插入到表中](#insert-data-to-table). -3. [将CatBoost集成到ClickHouse中](#integrate-catboost-into-clickhouse) （可跳过）。 -4. [从SQL运行模型推断](#run-model-inference). - -有关训练CatBoost模型的详细信息，请参阅 [训练和模型应用](https://catboost.ai/docs/features/training.html#training). - -您可以通过[RELOAD MODEL](https://clickhouse.com/docs/en/sql-reference/statements/system/#query_language-system-reload-model)与[RELOAD MODELS](https://clickhouse.com/docs/en/sql-reference/statements/system/#query_language-system-reload-models)语句来重载CatBoost模型。 - -## 先决条件 {#prerequisites} - -请先安装 [Docker](https://docs.docker.com/install/)。 - -!!! note "注" - [Docker](https://www.docker.com) 是一个软件平台，用户可以用Docker来创建独立于已有系统并集成了CatBoost和ClickHouse的容器。 - -在应用CatBoost模型之前: - -**1.** 从容器仓库拉取示例docker镜像 (https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) : - -``` bash -$ docker pull yandex/tutorial-catboost-clickhouse -``` - -此示例Docker镜像包含运行CatBoost和ClickHouse所需的所有内容：代码、运行时、库、环境变量和配置文件。 - -**2.** 确保已成功拉取Docker镜像: - -``` bash -$ docker image ls -REPOSITORY TAG IMAGE ID CREATED SIZE -yandex/tutorial-catboost-clickhouse latest 622e4d17945b 22 hours ago 1.37GB -``` - -**3.** 基于此镜像启动一个Docker容器: - -``` bash -$ docker run -it -p 8888:8888 yandex/tutorial-catboost-clickhouse -``` - -## 1. 创建数据表 {#create-table} - -为训练样本创建ClickHouse表: - -**1.** 在交互模式下启动ClickHouse控制台客户端: - -``` bash -$ clickhouse client -``` - -!!! note "注" - ClickHouse服务器已经在Docker容器内运行。 - -**2.** 使用以下命令创建表: - -``` sql -:) CREATE TABLE amazon_train -( - date Date MATERIALIZED today(), - ACTION UInt8, - RESOURCE UInt32, - MGR_ID UInt32, - ROLE_ROLLUP_1 UInt32, - ROLE_ROLLUP_2 UInt32, - ROLE_DEPTNAME UInt32, - ROLE_TITLE UInt32, - ROLE_FAMILY_DESC UInt32, - ROLE_FAMILY UInt32, - ROLE_CODE UInt32 -) -ENGINE = MergeTree ORDER BY date -``` - -**3.** 从ClickHouse控制台客户端退出: - -``` sql -:) exit -``` - -## 2. 将数据插入到表中 {#insert-data-to-table} - -插入数据: - -**1.** 运行以下命令: - -``` bash -$ clickhouse client --host 127.0.0.1 --query 'INSERT INTO amazon_train FORMAT CSVWithNames' < ~/amazon/train.csv -``` - -**2.** 在交互模式下启动ClickHouse控制台客户端: - -``` bash -$ clickhouse client -``` - -**3.** 确保数据已上传: - -``` sql -:) SELECT count() FROM amazon_train - -SELECT count() -FROM amazon_train - -+-count()-+ -| 65538 | -+-------+ -``` - -## 3. 将CatBoost集成到ClickHouse中 {#integrate-catboost-into-clickhouse} - -!!! note "注" - **可跳过。** 示例Docker映像已经包含了运行CatBoost和ClickHouse所需的所有内容。 - -为了将CatBoost集成进ClickHouse，需要进行如下步骤： - -**1.** 构建评估库。 - -评估CatBoost模型的最快方法是编译 `libcatboostmodel.` 库文件. - -有关如何构建库文件的详细信息，请参阅 [CatBoost文件](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). - -**2.** 创建一个新目录（位置与名称可随意指定）, 如 `data` 并将创建的库文件放入其中。 示例Docker镜像已经包含了库 `data/libcatboostmodel.so`. - -**3.** 创建一个新目录来放配置模型, 如 `models`. - -**4.** 创建一个模型配置文件，如 `models/amazon_model.xml`. - -**5.** 修改模型配置: - -``` xml - - - - catboost - - amazon - - /home/catboost/tutorial/catboost_model.bin - - 0 - - -``` - -**6.** 将CatBoost库文件的路径和模型配置添加到ClickHouse配置: - -``` xml - -/home/catboost/data/libcatboostmodel.so -/home/catboost/models/*_model.xml -``` - -## 4. 使用SQL调用预测模型 {#run-model-inference} - -为了测试模型是否正常，可以使用ClickHouse客户端 `$ clickhouse client`. - -让我们确保模型能正常工作: - -``` sql -:) SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) > 0 AS prediction, - ACTION AS target -FROM amazon_train -LIMIT 10 -``` - -!!! note "注" - 函数 [modelEvaluate](../sql-reference/functions/other-functions.md#function-modelevaluate) 会对多类别模型返回一个元组，其中包含每一类别的原始预测值。 - -执行预测: - -``` sql -:) SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) AS prediction, - 1. / (1 + exp(-prediction)) AS probability, - ACTION AS target -FROM amazon_train -LIMIT 10 -``` - -!!! note "注" - 查看函数说明 [exp()](../sql-reference/functions/math-functions.md) 。 - -让我们计算样本的LogLoss: - -``` sql -:) SELECT -avg(tg * log(prob) + (1 - tg) * log(1 - prob)) AS logloss -FROM -( - SELECT - modelEvaluate('amazon', - RESOURCE, - MGR_ID, - ROLE_ROLLUP_1, - ROLE_ROLLUP_2, - ROLE_DEPTNAME, - ROLE_TITLE, - ROLE_FAMILY_DESC, - ROLE_FAMILY, - ROLE_CODE) AS prediction, - 1. / (1. + exp(-prediction)) AS prob, - ACTION AS tg - FROM amazon_train -) -``` - -!!! note "注" - 查看函数说明 [avg()](../sql-reference/aggregate-functions/reference/avg.md#agg_function-avg) 和 [log()](../sql-reference/functions/math-functions.md) 。 - -[原始文章](https://clickhouse.com/docs/en/guides/apply_catboost_model/) diff --git a/docs/zh/guides/index.md b/docs/zh/guides/index.md index 5e535ea5736..00c4ae4def1 100644 --- a/docs/zh/guides/index.md +++ b/docs/zh/guides/index.md @@ -9,6 +9,5 @@ sidebar_label: ClickHouse指南 列出了如何使用 Clickhouse 解决各种任务的详细说明： - [关于简单集群设置的教程](../getting-started/tutorial.md) -- [在ClickHouse中应用CatBoost模型](apply-catboost-model.md) [原始文章](https://clickhouse.com/docs/en/guides/) diff --git a/docs/zh/sql-reference/statements/alter.md b/docs/zh/sql-reference/statements/alter.md index 2e143d3b654..23edfd633db 100644 --- a/docs/zh/sql-reference/statements/alter.md +++ b/docs/zh/sql-reference/statements/alter.md @@ -500,7 +500,7 @@ ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` ### 说明 {#alter-user-dscr} @@ -540,7 +540,7 @@ ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 ``` sql ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] [RENAME TO new_name] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` ## 修改row policy {#alter-row-policy-statement} @@ -584,7 +584,7 @@ ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] ``` sql ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name] [RENAME TO new_name] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` [Original article](https://clickhouse.com/docs/en/query_language/alter/) diff --git a/docs/zh/sql-reference/statements/alter/role.md b/docs/zh/sql-reference/statements/alter/role.md index e364571359f..3647f94e46e 100644 --- a/docs/zh/sql-reference/statements/alter/role.md +++ b/docs/zh/sql-reference/statements/alter/role.md @@ -13,5 +13,5 @@ sidebar_label: 角色 ``` sql ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] ``` diff --git a/docs/zh/sql-reference/statements/alter/settings-profile.md b/docs/zh/sql-reference/statements/alter/settings-profile.md index e4365b25c1a..e21f18f920b 100644 --- a/docs/zh/sql-reference/statements/alter/settings-profile.md +++ b/docs/zh/sql-reference/statements/alter/settings-profile.md @@ -13,5 +13,5 @@ sidebar_label: 配置文件设置 ``` sql ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` diff --git a/docs/zh/sql-reference/statements/system.md b/docs/zh/sql-reference/statements/system.md index d833887a9c6..3df00cf8854 100644 --- a/docs/zh/sql-reference/statements/system.md +++ b/docs/zh/sql-reference/statements/system.md @@ -6,38 +6,6 @@ sidebar_label: SYSTEM # SYSTEM Queries {#query-language-system} -- [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries) -- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) -- [RELOAD DICTIONARY](#query_language-system-reload-dictionary) -- [DROP DNS CACHE](#query_language-system-drop-dns-cache) -- [DROP MARK CACHE](#query_language-system-drop-mark-cache) -- [DROP UNCOMPRESSED CACHE](#query_language-system-drop-uncompressed-cache) -- [DROP COMPILED EXPRESSION CACHE](#query_language-system-drop-compiled-expression-cache) -- [DROP REPLICA](#query_language-system-drop-replica) -- [FLUSH LOGS](#query_language-system-flush_logs) -- [RELOAD CONFIG](#query_language-system-reload-config) -- [SHUTDOWN](#query_language-system-shutdown) -- [KILL](#query_language-system-kill) -- [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) -- [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) -- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) -- [STOP MERGES](#query_language-system-stop-merges) -- [START MERGES](#query_language-system-start-merges) -- [STOP TTL MERGES](#query_language-stop-ttl-merges) -- [START TTL MERGES](#query_language-start-ttl-merges) -- [STOP MOVES](#query_language-stop-moves) -- [START MOVES](#query_language-start-moves) -- [SYSTEM UNFREEZE](#query_language-system-unfreeze) -- [STOP FETCHES](#query_language-system-stop-fetches) -- [START FETCHES](#query_language-system-start-fetches) -- [STOP REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [START REPLICATED SENDS](#query_language-system-start-replicated-sends) -- [STOP REPLICATION QUEUES](#query_language-system-stop-replication-queues) -- [START REPLICATION QUEUES](#query_language-system-start-replication-queues) -- [SYNC REPLICA](#query_language-system-sync-replica) -- [RESTART REPLICA](#query_language-system-restart-replica) -- [RESTART REPLICAS](#query_language-system-restart-replicas) - ## RELOAD EMBEDDED DICTIONARIES\] {#query_language-system-reload-emdedded-dictionaries} 重新加载所有[内置字典](../../sql-reference/dictionaries/internal-dicts.md)。默认情况下内置字典是禁用的。 diff --git a/packages/clickhouse-client.yaml b/packages/clickhouse-client.yaml index 642d66f5475..459a09ee0b8 100644 --- a/packages/clickhouse-client.yaml +++ b/packages/clickhouse-client.yaml @@ -30,6 +30,10 @@ overrides: depends: - clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING} +deb: + fields: + Source: clickhouse + contents: - src: root/etc/clickhouse-client/config.xml dst: /etc/clickhouse-client/config.xml diff --git a/packages/clickhouse-common-static-dbg.yaml b/packages/clickhouse-common-static-dbg.yaml index 12a1594bd30..b2d2b3aaf26 100644 --- a/packages/clickhouse-common-static-dbg.yaml +++ b/packages/clickhouse-common-static-dbg.yaml @@ -20,6 +20,10 @@ description: | debugging symbols for clickhouse-common-static This package contains the debugging symbols for clickhouse-common. +deb: + fields: + Source: clickhouse + contents: - src: root/usr/lib/debug/usr/bin/clickhouse.debug dst: /usr/lib/debug/usr/bin/clickhouse.debug diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml index 527b6a24703..c77914d0d69 100644 --- a/packages/clickhouse-common-static.yaml +++ b/packages/clickhouse-common-static.yaml @@ -26,6 +26,10 @@ description: | that allows generating analytical data reports in real time. This package provides common files for both clickhouse server and client +deb: + fields: + Source: clickhouse + contents: - src: root/usr/bin/clickhouse dst: /usr/bin/clickhouse diff --git a/packages/clickhouse-keeper-dbg.yaml b/packages/clickhouse-keeper-dbg.yaml index 2c70b7ad4aa..a6be9ec9e97 100644 --- a/packages/clickhouse-keeper-dbg.yaml +++ b/packages/clickhouse-keeper-dbg.yaml @@ -14,6 +14,10 @@ description: | debugging symbols for clickhouse-keeper This package contains the debugging symbols for clickhouse-keeper. +deb: + fields: + Source: clickhouse + contents: - src: root/usr/lib/debug/usr/bin/clickhouse-keeper.debug dst: /usr/lib/debug/usr/bin/clickhouse-keeper.debug diff --git a/packages/clickhouse-keeper.yaml b/packages/clickhouse-keeper.yaml index e99ac30f944..7803729c469 100644 --- a/packages/clickhouse-keeper.yaml +++ b/packages/clickhouse-keeper.yaml @@ -22,6 +22,9 @@ description: | Static clickhouse-keeper binary A stand-alone clickhouse-keeper package +deb: + fields: + Source: clickhouse contents: - src: root/etc/clickhouse-keeper diff --git a/packages/clickhouse-server.init b/packages/clickhouse-server.init index 1695f6286b8..13aeffe13a7 100755 --- a/packages/clickhouse-server.init +++ b/packages/clickhouse-server.init @@ -47,9 +47,10 @@ CLICKHOUSE_PIDFILE="$CLICKHOUSE_PIDDIR/$PROGRAM.pid" # Some systems lack "flock" command -v flock >/dev/null && FLOCK=flock -# Override defaults from optional config file +# Override defaults from optional config file and export them automatically +set -a test -f /etc/default/clickhouse && . /etc/default/clickhouse - +set +a die() { diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml index 28995689754..a94ad1e9169 100644 --- a/packages/clickhouse-server.yaml +++ b/packages/clickhouse-server.yaml @@ -37,6 +37,10 @@ overrides: depends: - clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING} +deb: + fields: + Source: clickhouse + contents: - src: root/etc/clickhouse-server dst: /etc/clickhouse-server diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 3c0c0781de6..9cf7cb2b624 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -54,7 +54,7 @@ else () endif () if (NOT USE_MUSL) - option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to Library dictionary source" ${ENABLE_CLICKHOUSE_ALL}) + option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to external dynamically loaded libraries" ${ENABLE_CLICKHOUSE_ALL}) endif () # https://presentations.clickhouse.com/matemarketing_2020/ diff --git a/programs/benchmark/clickhouse-benchmark.cpp b/programs/benchmark/clickhouse-benchmark.cpp index 6bcb6e19b88..d58dd4dfbe7 100644 --- a/programs/benchmark/clickhouse-benchmark.cpp +++ b/programs/benchmark/clickhouse-benchmark.cpp @@ -1,2 +1 @@ extern int mainEntryClickHouseBenchmark(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseBenchmark(argc_, argv_); } diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 6506c23428a..5bd9d28d8e3 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -150,7 +150,7 @@ std::vector Client::loadWarningMessages() size_t rows = packet.block.rows(); for (size_t i = 0; i < rows; ++i) - messages.emplace_back(column.getDataAt(i).toString()); + messages.emplace_back(column[i].get()); } continue; diff --git a/programs/client/clickhouse-client.cpp b/programs/client/clickhouse-client.cpp index 8f7da31d617..3e489c5ff15 100644 --- a/programs/client/clickhouse-client.cpp +++ b/programs/client/clickhouse-client.cpp @@ -1,2 +1 @@ int mainEntryClickHouseClient(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseClient(argc_, argv_); } diff --git a/programs/compressor/clickhouse-compressor.cpp b/programs/compressor/clickhouse-compressor.cpp index f7d8611eac5..13a00ab6aff 100644 --- a/programs/compressor/clickhouse-compressor.cpp +++ b/programs/compressor/clickhouse-compressor.cpp @@ -1,2 +1 @@ int mainEntryClickHouseCompressor(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseCompressor(argc_, argv_); } diff --git a/programs/copier/clickhouse-copier.cpp b/programs/copier/clickhouse-copier.cpp index 653a0128aa4..4dabb01775b 100644 --- a/programs/copier/clickhouse-copier.cpp +++ b/programs/copier/clickhouse-copier.cpp @@ -1,2 +1 @@ int mainEntryClickHouseClusterCopier(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseClusterCopier(argc_, argv_); } diff --git a/programs/disks/clickhouse-disks.cpp b/programs/disks/clickhouse-disks.cpp index 184de86ee77..67ace2ddfd3 100644 --- a/programs/disks/clickhouse-disks.cpp +++ b/programs/disks/clickhouse-disks.cpp @@ -1,2 +1 @@ int mainEntryClickHouseDisks(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseDisks(argc_, argv_); } diff --git a/programs/extract-from-config/clickhouse-extract-from-config.cpp b/programs/extract-from-config/clickhouse-extract-from-config.cpp index 95d119b0dad..9a384c8f2dc 100644 --- a/programs/extract-from-config/clickhouse-extract-from-config.cpp +++ b/programs/extract-from-config/clickhouse-extract-from-config.cpp @@ -1,2 +1 @@ int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseExtractFromConfig(argc_, argv_); } diff --git a/programs/format/clickhouse-format.cpp b/programs/format/clickhouse-format.cpp index 6bda5c5e039..71f2a071312 100644 --- a/programs/format/clickhouse-format.cpp +++ b/programs/format/clickhouse-format.cpp @@ -1,2 +1 @@ int mainEntryClickHouseFormat(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseFormat(argc_, argv_); } diff --git a/programs/git-import/clickhouse-git-import.cpp b/programs/git-import/clickhouse-git-import.cpp index cfa06306604..57b66f3f77a 100644 --- a/programs/git-import/clickhouse-git-import.cpp +++ b/programs/git-import/clickhouse-git-import.cpp @@ -1,2 +1 @@ int mainEntryClickHouseGitImport(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseGitImport(argc_, argv_); } diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index 297e2d24c07..9b1bae947d2 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -446,8 +446,8 @@ int mainEntryClickHouseInstall(int argc, char ** argv) fs::path ulimits_file = ulimits_dir / fmt::format("{}.conf", user); fmt::print("Will set ulimits for {} user in {}.\n", user, ulimits_file.string()); std::string ulimits_content = fmt::format( - "{0}\tsoft\tnofile\t262144\n" - "{0}\thard\tnofile\t262144\n", user); + "{0}\tsoft\tnofile\t1048576\n" + "{0}\thard\tnofile\t1048576\n", user); fs::create_directories(ulimits_dir); diff --git a/programs/keeper-converter/clickhouse-keeper-converter.cpp b/programs/keeper-converter/clickhouse-keeper-converter.cpp index 3cb6f99f837..c84fc51f8cc 100644 --- a/programs/keeper-converter/clickhouse-keeper-converter.cpp +++ b/programs/keeper-converter/clickhouse-keeper-converter.cpp @@ -1,2 +1 @@ int mainEntryClickHouseKeeperConverter(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseKeeperConverter(argc_, argv_); } diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index 36e37eda91f..a5ad506abe6 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -103,6 +103,7 @@ if (BUILD_STANDALONE_KEEPER) # Remove some redundant dependencies target_compile_definitions (clickhouse-keeper PRIVATE -DKEEPER_STANDALONE_BUILD) + target_compile_definitions (clickhouse-keeper PUBLIC -DWITHOUT_TEXT_LOG) target_include_directories(clickhouse-keeper PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../src") # uses includes from src directory target_include_directories(clickhouse-keeper PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/../../src/Core/include") # uses some includes from core diff --git a/programs/library-bridge/CMakeLists.txt b/programs/library-bridge/CMakeLists.txt index 40cabacded4..1cacc391ca5 100644 --- a/programs/library-bridge/CMakeLists.txt +++ b/programs/library-bridge/CMakeLists.txt @@ -1,12 +1,15 @@ include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake) set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES + CatBoostLibraryHandler.cpp + CatBoostLibraryHandlerFactory.cpp ExternalDictionaryLibraryAPI.cpp ExternalDictionaryLibraryHandler.cpp ExternalDictionaryLibraryHandlerFactory.cpp LibraryBridge.cpp LibraryBridgeHandlerFactory.cpp LibraryBridgeHandlers.cpp + SharedLibrary.cpp library-bridge.cpp ) diff --git a/programs/library-bridge/CatBoostLibraryAPI.h b/programs/library-bridge/CatBoostLibraryAPI.h new file mode 100644 index 00000000000..9eaa0d17ff7 --- /dev/null +++ b/programs/library-bridge/CatBoostLibraryAPI.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +// Function pointer typedefs and names of libcatboost.so functions used by ClickHouse +struct CatBoostLibraryAPI +{ + using ModelCalcerHandle = void; + + using ModelCalcerCreateFunc = ModelCalcerHandle * (*)(); + static constexpr const char * ModelCalcerCreateName = "ModelCalcerCreate"; + + using ModelCalcerDeleteFunc = void (*)(ModelCalcerHandle *); + static constexpr const char * ModelCalcerDeleteName = "ModelCalcerDelete"; + + using GetErrorStringFunc = const char * (*)(); + static constexpr const char * GetErrorStringName = "GetErrorString"; + + using LoadFullModelFromFileFunc = bool (*)(ModelCalcerHandle *, const char *); + static constexpr const char * LoadFullModelFromFileName = "LoadFullModelFromFile"; + + using CalcModelPredictionFlatFunc = bool (*)(ModelCalcerHandle *, size_t, const float **, size_t, double *, size_t); + static constexpr const char * CalcModelPredictionFlatName = "CalcModelPredictionFlat"; + + using CalcModelPredictionFunc = bool (*)(ModelCalcerHandle *, size_t, const float **, size_t, const char ***, size_t, double *, size_t); + static constexpr const char * CalcModelPredictionName = "CalcModelPrediction"; + + using CalcModelPredictionWithHashedCatFeaturesFunc = bool (*)(ModelCalcerHandle *, size_t, const float **, size_t, const int **, size_t, double *, size_t); + static constexpr const char * CalcModelPredictionWithHashedCatFeaturesName = "CalcModelPredictionWithHashedCatFeatures"; + + using GetStringCatFeatureHashFunc = int (*)(const char *, size_t); + static constexpr const char * GetStringCatFeatureHashName = "GetStringCatFeatureHash"; + + using GetIntegerCatFeatureHashFunc = int (*)(uint64_t); + static constexpr const char * GetIntegerCatFeatureHashName = "GetIntegerCatFeatureHash"; + + using GetFloatFeaturesCountFunc = size_t (*)(ModelCalcerHandle *); + static constexpr const char * GetFloatFeaturesCountName = "GetFloatFeaturesCount"; + + using GetCatFeaturesCountFunc = size_t (*)(ModelCalcerHandle *); + static constexpr const char * GetCatFeaturesCountName = "GetCatFeaturesCount"; + + using GetTreeCountFunc = size_t (*)(ModelCalcerHandle *); + static constexpr const char * GetTreeCountName = "GetTreeCount"; + + using GetDimensionsCountFunc = size_t (*)(ModelCalcerHandle *); + static constexpr const char * GetDimensionsCountName = "GetDimensionsCount"; +}; diff --git a/programs/library-bridge/CatBoostLibraryHandler.cpp b/programs/library-bridge/CatBoostLibraryHandler.cpp new file mode 100644 index 00000000000..4fe539a53b2 --- /dev/null +++ b/programs/library-bridge/CatBoostLibraryHandler.cpp @@ -0,0 +1,389 @@ +#include "CatBoostLibraryHandler.h" + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int CANNOT_APPLY_CATBOOST_MODEL; + extern const int CANNOT_LOAD_CATBOOST_MODEL; + extern const int LOGICAL_ERROR; +} + +CatBoostLibraryHandler::APIHolder::APIHolder(SharedLibrary & lib) +{ + ModelCalcerCreate = lib.get(CatBoostLibraryAPI::ModelCalcerCreateName); + ModelCalcerDelete = lib.get(CatBoostLibraryAPI::ModelCalcerDeleteName); + GetErrorString = lib.get(CatBoostLibraryAPI::GetErrorStringName); + LoadFullModelFromFile = lib.get(CatBoostLibraryAPI::LoadFullModelFromFileName); + CalcModelPredictionFlat = lib.get(CatBoostLibraryAPI::CalcModelPredictionFlatName); + CalcModelPrediction = lib.get(CatBoostLibraryAPI::CalcModelPredictionName); + CalcModelPredictionWithHashedCatFeatures = lib.get(CatBoostLibraryAPI::CalcModelPredictionWithHashedCatFeaturesName); + GetStringCatFeatureHash = lib.get(CatBoostLibraryAPI::GetStringCatFeatureHashName); + GetIntegerCatFeatureHash = lib.get(CatBoostLibraryAPI::GetIntegerCatFeatureHashName); + GetFloatFeaturesCount = lib.get(CatBoostLibraryAPI::GetFloatFeaturesCountName); + GetCatFeaturesCount = lib.get(CatBoostLibraryAPI::GetCatFeaturesCountName); + GetTreeCount = lib.tryGet(CatBoostLibraryAPI::GetTreeCountName); + GetDimensionsCount = lib.tryGet(CatBoostLibraryAPI::GetDimensionsCountName); +} + +CatBoostLibraryHandler::CatBoostLibraryHandler( + const std::string & library_path, + const std::string & model_path) + : loading_start_time(std::chrono::system_clock::now()) + , library(std::make_shared(library_path)) + , api(*library) +{ + model_calcer_handle = api.ModelCalcerCreate(); + + if (!api.LoadFullModelFromFile(model_calcer_handle, model_path.c_str())) + { + throw Exception(ErrorCodes::CANNOT_LOAD_CATBOOST_MODEL, + "Cannot load CatBoost model: {}", api.GetErrorString()); + } + + float_features_count = api.GetFloatFeaturesCount(model_calcer_handle); + cat_features_count = api.GetCatFeaturesCount(model_calcer_handle); + + tree_count = 1; + if (api.GetDimensionsCount) + tree_count = api.GetDimensionsCount(model_calcer_handle); + + loading_duration = std::chrono::duration_cast(std::chrono::system_clock::now() - loading_start_time); +} + +CatBoostLibraryHandler::~CatBoostLibraryHandler() +{ + api.ModelCalcerDelete(model_calcer_handle); +} + +std::chrono::system_clock::time_point CatBoostLibraryHandler::getLoadingStartTime() const +{ + return loading_start_time; +} + +std::chrono::milliseconds CatBoostLibraryHandler::getLoadingDuration() const +{ + return loading_duration; +} + +namespace +{ + +/// Buffer should be allocated with features_count * column->size() elements. +/// Place column elements in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] +template +void placeColumnAsNumber(const IColumn * column, T * buffer, size_t features_count) +{ + size_t size = column->size(); + FieldVisitorConvertToNumber visitor; + for (size_t i = 0; i < size; ++i) + { + /// TODO: Replace with column visitor. + Field field; + column->get(i, field); + *buffer = applyVisitor(visitor, field); + buffer += features_count; + } +} + +/// Buffer should be allocated with features_count * column->size() elements. +/// Place string pointers in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] +void placeStringColumn(const ColumnString & column, const char ** buffer, size_t features_count) +{ + size_t size = column.size(); + for (size_t i = 0; i < size; ++i) + { + *buffer = const_cast(column.getDataAt(i).data); + buffer += features_count; + } +} + +/// Buffer should be allocated with features_count * column->size() elements. +/// Place string pointers in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] +/// Returns PODArray which holds data (because ColumnFixedString doesn't store terminating zero). +PODArray placeFixedStringColumn(const ColumnFixedString & column, const char ** buffer, size_t features_count) +{ + size_t size = column.size(); + size_t str_size = column.getN(); + PODArray data(size * (str_size + 1)); + char * data_ptr = data.data(); + + for (size_t i = 0; i < size; ++i) + { + auto ref = column.getDataAt(i); + memcpy(data_ptr, ref.data, ref.size); + data_ptr[ref.size] = 0; + *buffer = data_ptr; + data_ptr += ref.size + 1; + buffer += features_count; + } + + return data; +} + +/// Place columns into buffer, returns column which holds placed data. Buffer should contains column->size() values. +template +ColumnPtr placeNumericColumns(const ColumnRawPtrs & columns, size_t offset, size_t size, const T** buffer) +{ + if (size == 0) + return nullptr; + + size_t column_size = columns[offset]->size(); + auto data_column = ColumnVector::create(size * column_size); + T * data = data_column->getData().data(); + for (size_t i = 0; i < size; ++i) + { + const auto * column = columns[offset + i]; + if (column->isNumeric()) + placeColumnAsNumber(column, data + i, size); + } + + for (size_t i = 0; i < column_size; ++i) + { + *buffer = data; + ++buffer; + data += size; + } + + return data_column; +} + +/// Place columns into buffer, returns data which was used for fixed string columns. +/// Buffer should contains column->size() values, each value contains size strings. +std::vector> placeStringColumns(const ColumnRawPtrs & columns, size_t offset, size_t size, const char ** buffer) +{ + if (size == 0) + return {}; + + std::vector> data; + for (size_t i = 0; i < size; ++i) + { + const auto * column = columns[offset + i]; + if (const auto * column_string = typeid_cast(column)) + placeStringColumn(*column_string, buffer + i, size); + else if (const auto * column_fixed_string = typeid_cast(column)) + data.push_back(placeFixedStringColumn(*column_fixed_string, buffer + i, size)); + else + throw Exception("Cannot place string column.", ErrorCodes::LOGICAL_ERROR); + } + + return data; +} + +/// buffer[column_size * cat_features_count] -> char * => cat_features[column_size][cat_features_count] -> char * +void fillCatFeaturesBuffer( + const char *** cat_features, const char ** buffer, + size_t column_size, size_t cat_features_count) +{ + for (size_t i = 0; i < column_size; ++i) + { + *cat_features = buffer; + ++cat_features; + buffer += cat_features_count; + } +} + +/// Calc hash for string cat feature at ps positions. +template +void calcStringHashes(const Column * column, size_t ps, const int ** buffer, const CatBoostLibraryHandler::APIHolder & api) +{ + size_t column_size = column->size(); + for (size_t j = 0; j < column_size; ++j) + { + auto ref = column->getDataAt(j); + const_cast(*buffer)[ps] = api.GetStringCatFeatureHash(ref.data, ref.size); + ++buffer; + } +} + +/// Calc hash for int cat feature at ps position. Buffer at positions ps should contains unhashed values. +void calcIntHashes(size_t column_size, size_t ps, const int ** buffer, const CatBoostLibraryHandler::APIHolder & api) +{ + for (size_t j = 0; j < column_size; ++j) + { + const_cast(*buffer)[ps] = api.GetIntegerCatFeatureHash((*buffer)[ps]); + ++buffer; + } +} + +/// buffer contains column->size() rows and size columns. +/// For int cat features calc hash inplace. +/// For string cat features calc hash from column rows. +void calcHashes(const ColumnRawPtrs & columns, size_t offset, size_t size, const int ** buffer, const CatBoostLibraryHandler::APIHolder & api) +{ + if (size == 0) + return; + size_t column_size = columns[offset]->size(); + + std::vector> data; + for (size_t i = 0; i < size; ++i) + { + const auto * column = columns[offset + i]; + if (const auto * column_string = typeid_cast(column)) + calcStringHashes(column_string, i, buffer, api); + else if (const auto * column_fixed_string = typeid_cast(column)) + calcStringHashes(column_fixed_string, i, buffer, api); + else + calcIntHashes(column_size, i, buffer, api); + } +} + +} + +/// Convert values to row-oriented format and call evaluation function from CatBoost wrapper api. +/// * CalcModelPredictionFlat if no cat features +/// * CalcModelPrediction if all cat features are strings +/// * CalcModelPredictionWithHashedCatFeatures if has int cat features. +ColumnFloat64::MutablePtr CatBoostLibraryHandler::evalImpl( + const ColumnRawPtrs & columns, + bool cat_features_are_strings) const +{ + std::string error_msg = "Error occurred while applying CatBoost model: "; + size_t column_size = columns.front()->size(); + + auto result = ColumnFloat64::create(column_size * tree_count); + auto * result_buf = result->getData().data(); + + if (!column_size) + return result; + + /// Prepare float features. + PODArray float_features(column_size); + auto * float_features_buf = float_features.data(); + /// Store all float data into single column. float_features is a list of pointers to it. + auto float_features_col = placeNumericColumns(columns, 0, float_features_count, float_features_buf); + + if (cat_features_count == 0) + { + if (!api.CalcModelPredictionFlat(model_calcer_handle, column_size, + float_features_buf, float_features_count, + result_buf, column_size * tree_count)) + { + + throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); + } + return result; + } + + /// Prepare cat features. + if (cat_features_are_strings) + { + /// cat_features_holder stores pointers to ColumnString data or fixed_strings_data. + PODArray cat_features_holder(cat_features_count * column_size); + PODArray cat_features(column_size); + auto * cat_features_buf = cat_features.data(); + + fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size, cat_features_count); + /// Fixed strings are stored without termination zero, so have to copy data into fixed_strings_data. + auto fixed_strings_data = placeStringColumns(columns, float_features_count, + cat_features_count, cat_features_holder.data()); + + if (!api.CalcModelPrediction(model_calcer_handle, column_size, + float_features_buf, float_features_count, + cat_features_buf, cat_features_count, + result_buf, column_size * tree_count)) + { + throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); + } + } + else + { + PODArray cat_features(column_size); + auto * cat_features_buf = cat_features.data(); + auto cat_features_col = placeNumericColumns(columns, float_features_count, + cat_features_count, cat_features_buf); + calcHashes(columns, float_features_count, cat_features_count, cat_features_buf, api); + if (!api.CalcModelPredictionWithHashedCatFeatures( + model_calcer_handle, column_size, + float_features_buf, float_features_count, + cat_features_buf, cat_features_count, + result_buf, column_size * tree_count)) + { + throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); + } + } + + return result; +} + +size_t CatBoostLibraryHandler::getTreeCount() const +{ + std::lock_guard lock(mutex); + return tree_count; +} + +ColumnPtr CatBoostLibraryHandler::evaluate(const ColumnRawPtrs & columns) const +{ + std::lock_guard lock(mutex); + + if (columns.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Got empty columns list for CatBoost model."); + + if (columns.size() != float_features_count + cat_features_count) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Number of columns is different with number of features: columns size {} float features size {} + cat features size {}", + columns.size(), + float_features_count, + cat_features_count); + + for (size_t i = 0; i < float_features_count; ++i) + { + if (!columns[i]->isNumeric()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} should be numeric to make float feature.", i); + } + } + + bool cat_features_are_strings = true; + for (size_t i = float_features_count; i < float_features_count + cat_features_count; ++i) + { + const auto * column = columns[i]; + if (column->isNumeric()) + { + cat_features_are_strings = false; + } + else if (!(typeid_cast(column) + || typeid_cast(column))) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} should be numeric or string.", i); + } + } + + auto result = evalImpl(columns, cat_features_are_strings); + + if (tree_count == 1) + return result; + + size_t column_size = columns.front()->size(); + auto * result_buf = result->getData().data(); + + /// Multiple trees case. Copy data to several columns. + MutableColumns mutable_columns(tree_count); + std::vector column_ptrs(tree_count); + for (size_t i = 0; i < tree_count; ++i) + { + auto col = ColumnFloat64::create(column_size); + column_ptrs[i] = col->getData().data(); + mutable_columns[i] = std::move(col); + } + + Float64 * data = result_buf; + for (size_t row = 0; row < column_size; ++row) + { + for (size_t i = 0; i < tree_count; ++i) + { + *column_ptrs[i] = *data; + ++column_ptrs[i]; + ++data; + } + } + + return ColumnTuple::create(std::move(mutable_columns)); +} + +} diff --git a/programs/library-bridge/CatBoostLibraryHandler.h b/programs/library-bridge/CatBoostLibraryHandler.h new file mode 100644 index 00000000000..a20e1412552 --- /dev/null +++ b/programs/library-bridge/CatBoostLibraryHandler.h @@ -0,0 +1,79 @@ +#pragma once + +#include "CatBoostLibraryAPI.h" + +#include +#include +#include +#include +#include +#include +#include "SharedLibrary.h" + +#include +#include + + +namespace DB +{ + +/// Abstracts access to the CatBoost shared library. +class CatBoostLibraryHandler +{ +public: + /// Holds pointers to CatBoost library functions + struct APIHolder + { + explicit APIHolder(SharedLibrary & lib); + + // NOLINTBEGIN(readability-identifier-naming) + CatBoostLibraryAPI::ModelCalcerCreateFunc ModelCalcerCreate; + CatBoostLibraryAPI::ModelCalcerDeleteFunc ModelCalcerDelete; + CatBoostLibraryAPI::GetErrorStringFunc GetErrorString; + CatBoostLibraryAPI::LoadFullModelFromFileFunc LoadFullModelFromFile; + CatBoostLibraryAPI::CalcModelPredictionFlatFunc CalcModelPredictionFlat; + CatBoostLibraryAPI::CalcModelPredictionFunc CalcModelPrediction; + CatBoostLibraryAPI::CalcModelPredictionWithHashedCatFeaturesFunc CalcModelPredictionWithHashedCatFeatures; + CatBoostLibraryAPI::GetStringCatFeatureHashFunc GetStringCatFeatureHash; + CatBoostLibraryAPI::GetIntegerCatFeatureHashFunc GetIntegerCatFeatureHash; + CatBoostLibraryAPI::GetFloatFeaturesCountFunc GetFloatFeaturesCount; + CatBoostLibraryAPI::GetCatFeaturesCountFunc GetCatFeaturesCount; + CatBoostLibraryAPI::GetTreeCountFunc GetTreeCount; + CatBoostLibraryAPI::GetDimensionsCountFunc GetDimensionsCount; + // NOLINTEND(readability-identifier-naming) + }; + + CatBoostLibraryHandler( + const String & library_path, + const String & model_path); + + ~CatBoostLibraryHandler(); + + std::chrono::system_clock::time_point getLoadingStartTime() const; + std::chrono::milliseconds getLoadingDuration() const; + + size_t getTreeCount() const; + + ColumnPtr evaluate(const ColumnRawPtrs & columns) const; + +private: + std::chrono::system_clock::time_point loading_start_time; + std::chrono::milliseconds loading_duration; + + const SharedLibraryPtr library; + const APIHolder api; + + mutable std::mutex mutex; + + CatBoostLibraryAPI::ModelCalcerHandle * model_calcer_handle TSA_GUARDED_BY(mutex) TSA_PT_GUARDED_BY(mutex); + + size_t float_features_count TSA_GUARDED_BY(mutex); + size_t cat_features_count TSA_GUARDED_BY(mutex); + size_t tree_count TSA_GUARDED_BY(mutex); + + ColumnFloat64::MutablePtr evalImpl(const ColumnRawPtrs & columns, bool cat_features_are_strings) const TSA_REQUIRES(mutex); +}; + +using CatBoostLibraryHandlerPtr = std::shared_ptr; + +} diff --git a/programs/library-bridge/CatBoostLibraryHandlerFactory.cpp b/programs/library-bridge/CatBoostLibraryHandlerFactory.cpp new file mode 100644 index 00000000000..6ee078f6c5c --- /dev/null +++ b/programs/library-bridge/CatBoostLibraryHandlerFactory.cpp @@ -0,0 +1,80 @@ +#include "CatBoostLibraryHandlerFactory.h" + +#include + + +namespace DB +{ + +CatBoostLibraryHandlerFactory & CatBoostLibraryHandlerFactory::instance() +{ + static CatBoostLibraryHandlerFactory instance; + return instance; +} + +CatBoostLibraryHandlerFactory::CatBoostLibraryHandlerFactory() + : log(&Poco::Logger::get("CatBoostLibraryHandlerFactory")) +{ +} + +CatBoostLibraryHandlerPtr CatBoostLibraryHandlerFactory::tryGetModel(const String & model_path, const String & library_path, bool create_if_not_found) +{ + std::lock_guard lock(mutex); + + auto handler = library_handlers.find(model_path); + bool found = (handler != library_handlers.end()); + + if (found) + return handler->second; + else + { + if (create_if_not_found) + { + auto new_handler = std::make_shared(library_path, model_path); + library_handlers.emplace(model_path, new_handler); + LOG_DEBUG(log, "Loaded catboost library handler for model path '{}'", model_path); + return new_handler; + } + return nullptr; + } +} + +void CatBoostLibraryHandlerFactory::removeModel(const String & model_path) +{ + std::lock_guard lock(mutex); + + bool deleted = library_handlers.erase(model_path); + if (!deleted) + { + LOG_DEBUG(log, "Cannot unload catboost library handler for model path '{}'", model_path); + return; + } + LOG_DEBUG(log, "Unloaded catboost library handler for model path '{}'", model_path); +} + +void CatBoostLibraryHandlerFactory::removeAllModels() +{ + std::lock_guard lock(mutex); + library_handlers.clear(); + LOG_DEBUG(log, "Unloaded all catboost library handlers"); +} + +ExternalModelInfos CatBoostLibraryHandlerFactory::getModelInfos() +{ + std::lock_guard lock(mutex); + + ExternalModelInfos result; + + for (const auto & handler : library_handlers) + result.push_back({ + .model_path = handler.first, + .model_type = "catboost", + .loading_start_time = handler.second->getLoadingStartTime(), + .loading_duration = handler.second->getLoadingDuration() + }); + + return result; + +} + +} diff --git a/programs/library-bridge/CatBoostLibraryHandlerFactory.h b/programs/library-bridge/CatBoostLibraryHandlerFactory.h new file mode 100644 index 00000000000..6ba3fe84ec9 --- /dev/null +++ b/programs/library-bridge/CatBoostLibraryHandlerFactory.h @@ -0,0 +1,37 @@ +#pragma once + +#include "CatBoostLibraryHandler.h" + +#include +#include + +#include +#include +#include + + +namespace DB +{ + +class CatBoostLibraryHandlerFactory final : private boost::noncopyable +{ +public: + static CatBoostLibraryHandlerFactory & instance(); + + CatBoostLibraryHandlerFactory(); + + CatBoostLibraryHandlerPtr tryGetModel(const String & model_path, const String & library_path, bool create_if_not_found); + + void removeModel(const String & model_path); + void removeAllModels(); + + ExternalModelInfos getModelInfos(); + +private: + /// map: model path --> catboost library handler + std::unordered_map library_handlers TSA_GUARDED_BY(mutex); + std::mutex mutex; + Poco::Logger * log; +}; + +} diff --git a/programs/library-bridge/ExternalDictionaryLibraryHandler.h b/programs/library-bridge/ExternalDictionaryLibraryHandler.h index 7713e9a6830..77c9b9fdf39 100644 --- a/programs/library-bridge/ExternalDictionaryLibraryHandler.h +++ b/programs/library-bridge/ExternalDictionaryLibraryHandler.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "SharedLibrary.h" #include #include "ExternalDictionaryLibraryUtils.h" @@ -50,6 +50,6 @@ private: void * lib_data; }; -using SharedLibraryHandlerPtr = std::shared_ptr; +using ExternalDictionaryLibraryHandlerPtr = std::shared_ptr; } diff --git a/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp index ffa5ff6f493..6acd9af20ed 100644 --- a/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp +++ b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp @@ -1,37 +1,40 @@ #include "ExternalDictionaryLibraryHandlerFactory.h" +#include namespace DB { -SharedLibraryHandlerPtr ExternalDictionaryLibraryHandlerFactory::get(const std::string & dictionary_id) +ExternalDictionaryLibraryHandlerPtr ExternalDictionaryLibraryHandlerFactory::get(const String & dictionary_id) { std::lock_guard lock(mutex); - auto library_handler = library_handlers.find(dictionary_id); - - if (library_handler != library_handlers.end()) - return library_handler->second; + if (auto handler = library_handlers.find(dictionary_id); handler != library_handlers.end()) + return handler->second; return nullptr; } void ExternalDictionaryLibraryHandlerFactory::create( - const std::string & dictionary_id, - const std::string & library_path, - const std::vector & library_settings, + const String & dictionary_id, + const String & library_path, + const std::vector & library_settings, const Block & sample_block, - const std::vector & attributes_names) + const std::vector & attributes_names) { std::lock_guard lock(mutex); - if (!library_handlers.contains(dictionary_id)) - library_handlers.emplace(std::make_pair(dictionary_id, std::make_shared(library_path, library_settings, sample_block, attributes_names))); - else + + if (library_handlers.contains(dictionary_id)) + { LOG_WARNING(&Poco::Logger::get("ExternalDictionaryLibraryHandlerFactory"), "Library handler with dictionary id {} already exists", dictionary_id); + return; + } + + library_handlers.emplace(std::make_pair(dictionary_id, std::make_shared(library_path, library_settings, sample_block, attributes_names))); } -bool ExternalDictionaryLibraryHandlerFactory::clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id) +bool ExternalDictionaryLibraryHandlerFactory::clone(const String & from_dictionary_id, const String & to_dictionary_id) { std::lock_guard lock(mutex); auto from_library_handler = library_handlers.find(from_dictionary_id); @@ -45,7 +48,7 @@ bool ExternalDictionaryLibraryHandlerFactory::clone(const std::string & from_dic } -bool ExternalDictionaryLibraryHandlerFactory::remove(const std::string & dictionary_id) +bool ExternalDictionaryLibraryHandlerFactory::remove(const String & dictionary_id) { std::lock_guard lock(mutex); /// extDict_libDelete is called in destructor. diff --git a/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h index d821270c474..3dfafd82a0f 100644 --- a/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h +++ b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h @@ -17,22 +17,22 @@ class ExternalDictionaryLibraryHandlerFactory final : private boost::noncopyable public: static ExternalDictionaryLibraryHandlerFactory & instance(); - SharedLibraryHandlerPtr get(const std::string & dictionary_id); + ExternalDictionaryLibraryHandlerPtr get(const String & dictionary_id); void create( - const std::string & dictionary_id, - const std::string & library_path, - const std::vector & library_settings, + const String & dictionary_id, + const String & library_path, + const std::vector & library_settings, const Block & sample_block, - const std::vector & attributes_names); + const std::vector & attributes_names); - bool clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id); + bool clone(const String & from_dictionary_id, const String & to_dictionary_id); - bool remove(const std::string & dictionary_id); + bool remove(const String & dictionary_id); private: /// map: dict_id -> sharedLibraryHandler - std::unordered_map library_handlers TSA_GUARDED_BY(mutex); + std::unordered_map library_handlers TSA_GUARDED_BY(mutex); std::mutex mutex; }; diff --git a/programs/library-bridge/LibraryBridgeHandlerFactory.cpp b/programs/library-bridge/LibraryBridgeHandlerFactory.cpp index f8f6a23e1be..4af1f8355e8 100644 --- a/programs/library-bridge/LibraryBridgeHandlerFactory.cpp +++ b/programs/library-bridge/LibraryBridgeHandlerFactory.cpp @@ -27,12 +27,16 @@ std::unique_ptr LibraryBridgeHandlerFactory::createRequestHa { if (uri.getPath() == "/extdict_ping") return std::make_unique(keep_alive_timeout, getContext()); + else if (uri.getPath() == "/catboost_ping") + return std::make_unique(keep_alive_timeout, getContext()); } if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) { if (uri.getPath() == "/extdict_request") return std::make_unique(keep_alive_timeout, getContext()); + else if (uri.getPath() == "/catboost_request") + return std::make_unique(keep_alive_timeout, getContext()); } return nullptr; diff --git a/programs/library-bridge/LibraryBridgeHandlers.cpp b/programs/library-bridge/LibraryBridgeHandlers.cpp index a28148bd1f7..ab81472be88 100644 --- a/programs/library-bridge/LibraryBridgeHandlers.cpp +++ b/programs/library-bridge/LibraryBridgeHandlers.cpp @@ -1,24 +1,32 @@ #include "LibraryBridgeHandlers.h" + +#include "CatBoostLibraryHandler.h" +#include "CatBoostLibraryHandlerFactory.h" +#include "ExternalDictionaryLibraryHandler.h" #include "ExternalDictionaryLibraryHandlerFactory.h" #include -#include -#include +#include #include #include +#include +#include #include #include -#include #include -#include -#include -#include #include #include +#include +#include #include #include +#include #include -#include +#include +#include +#include +#include +#include namespace DB @@ -31,7 +39,7 @@ namespace ErrorCodes namespace { - void processError(HTTPServerResponse & response, const std::string & message) + void processError(HTTPServerResponse & response, const String & message) { response.setStatusAndReason(HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); @@ -41,7 +49,7 @@ namespace LOG_WARNING(&Poco::Logger::get("LibraryBridge"), fmt::runtime(message)); } - std::shared_ptr parseColumns(std::string && column_string) + std::shared_ptr parseColumns(String && column_string) { auto sample_block = std::make_shared(); auto names_and_types = NamesAndTypesList::parse(column_string); @@ -59,10 +67,10 @@ namespace return ids; } - std::vector parseNamesFromBinary(const std::string & names_string) + std::vector parseNamesFromBinary(const String & names_string) { ReadBufferFromString buf(names_string); - std::vector names; + std::vector names; readVectorBinary(names, buf); return names; } @@ -79,13 +87,15 @@ static void writeData(Block data, OutputFormatPtr format) executor.execute(); } + ExternalDictionaryLibraryBridgeRequestHandler::ExternalDictionaryLibraryBridgeRequestHandler(size_t keep_alive_timeout_, ContextPtr context_) : WithContext(context_) - , log(&Poco::Logger::get("ExternalDictionaryLibraryBridgeRequestHandler")) , keep_alive_timeout(keep_alive_timeout_) + , log(&Poco::Logger::get("ExternalDictionaryLibraryBridgeRequestHandler")) { } + void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { LOG_TRACE(log, "Request URI: {}", request.getURI()); @@ -97,7 +107,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ version = 0; /// assumed version for too old servers which do not send a version else { - String version_str = params.get("version"); + const String & version_str = params.get("version"); if (!tryParse(version, version_str)) { processError(response, "Unable to parse 'version' string in request URL: '" + version_str + "' Check if the server and library-bridge have the same version."); @@ -124,8 +134,8 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - std::string method = params.get("method"); - std::string dictionary_id = params.get("dictionary_id"); + const String & method = params.get("method"); + const String & dictionary_id = params.get("dictionary_id"); LOG_TRACE(log, "Library method: '{}', dictionary id: {}", method, dictionary_id); WriteBufferFromHTTPServerResponse out(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout); @@ -141,7 +151,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - std::string from_dictionary_id = params.get("from_dictionary_id"); + const String & from_dictionary_id = params.get("from_dictionary_id"); bool cloned = false; cloned = ExternalDictionaryLibraryHandlerFactory::instance().clone(from_dictionary_id, dictionary_id); @@ -166,7 +176,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - std::string library_path = params.get("library_path"); + const String & library_path = params.get("library_path"); if (!params.has("library_settings")) { @@ -174,10 +184,10 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - const auto & settings_string = params.get("library_settings"); + const String & settings_string = params.get("library_settings"); LOG_DEBUG(log, "Parsing library settings from binary string"); - std::vector library_settings = parseNamesFromBinary(settings_string); + std::vector library_settings = parseNamesFromBinary(settings_string); /// Needed for library dictionary if (!params.has("attributes_names")) @@ -186,10 +196,10 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - const auto & attributes_string = params.get("attributes_names"); + const String & attributes_string = params.get("attributes_names"); LOG_DEBUG(log, "Parsing attributes names from binary string"); - std::vector attributes_names = parseNamesFromBinary(attributes_string); + std::vector attributes_names = parseNamesFromBinary(attributes_string); /// Needed to parse block from binary string format if (!params.has("sample_block")) @@ -197,7 +207,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ processError(response, "No 'sample_block' in request URL"); return; } - std::string sample_block_string = params.get("sample_block"); + String sample_block_string = params.get("sample_block"); std::shared_ptr sample_block; try @@ -297,7 +307,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ return; } - std::string requested_block_string = params.get("requested_block_sample"); + String requested_block_string = params.get("requested_block_sample"); std::shared_ptr requested_sample_block; try @@ -332,7 +342,8 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ } else { - LOG_WARNING(log, "Unknown library method: '{}'", method); + processError(response, "Unknown library method '" + method + "'"); + LOG_ERROR(log, "Unknown library method: '{}'", method); } } catch (...) @@ -362,6 +373,7 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ } } + ExternalDictionaryLibraryBridgeExistsHandler::ExternalDictionaryLibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_) : WithContext(context_) , keep_alive_timeout(keep_alive_timeout_) @@ -369,6 +381,7 @@ ExternalDictionaryLibraryBridgeExistsHandler::ExternalDictionaryLibraryBridgeExi { } + void ExternalDictionaryLibraryBridgeExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { try @@ -382,7 +395,7 @@ void ExternalDictionaryLibraryBridgeExistsHandler::handleRequest(HTTPServerReque return; } - std::string dictionary_id = params.get("dictionary_id"); + const String & dictionary_id = params.get("dictionary_id"); auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id); @@ -399,4 +412,230 @@ void ExternalDictionaryLibraryBridgeExistsHandler::handleRequest(HTTPServerReque } +CatBoostLibraryBridgeRequestHandler::CatBoostLibraryBridgeRequestHandler( + size_t keep_alive_timeout_, ContextPtr context_) + : WithContext(context_) + , keep_alive_timeout(keep_alive_timeout_) + , log(&Poco::Logger::get("CatBoostLibraryBridgeRequestHandler")) +{ +} + + +void CatBoostLibraryBridgeRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) +{ + LOG_TRACE(log, "Request URI: {}", request.getURI()); + HTMLForm params(getContext()->getSettingsRef(), request); + + size_t version; + + if (!params.has("version")) + version = 0; /// assumed version for too old servers which do not send a version + else + { + const String & version_str = params.get("version"); + if (!tryParse(version, version_str)) + { + processError(response, "Unable to parse 'version' string in request URL: '" + version_str + "' Check if the server and library-bridge have the same version."); + return; + } + } + + if (version != LIBRARY_BRIDGE_PROTOCOL_VERSION) + { + /// backwards compatibility is considered unnecessary for now, just let the user know that the server and the bridge must be upgraded together + processError(response, "Server and library-bridge have different versions: '" + std::to_string(version) + "' vs. '" + std::to_string(LIBRARY_BRIDGE_PROTOCOL_VERSION) + "'"); + return; + } + if (!params.has("method")) + { + processError(response, "No 'method' in request URL"); + return; + } + + const String & method = params.get("method"); + + LOG_TRACE(log, "Library method: '{}'", method); + WriteBufferFromHTTPServerResponse out(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout); + + try + { + if (method == "catboost_list") + { + ExternalModelInfos model_infos = CatBoostLibraryHandlerFactory::instance().getModelInfos(); + + writeIntBinary(static_cast(model_infos.size()), out); + + for (const auto & info : model_infos) + { + writeStringBinary(info.model_path, out); + writeStringBinary(info.model_type, out); + + UInt64 t = std::chrono::system_clock::to_time_t(info.loading_start_time); + writeIntBinary(t, out); + + t = info.loading_duration.count(); + writeIntBinary(t, out); + + } + } + else if (method == "catboost_removeModel") + { + auto & read_buf = request.getStream(); + params.read(read_buf); + + if (!params.has("model_path")) + { + processError(response, "No 'model_path' in request URL"); + return; + } + + const String & model_path = params.get("model_path"); + + CatBoostLibraryHandlerFactory::instance().removeModel(model_path); + + String res = "1"; + writeStringBinary(res, out); + } + else if (method == "catboost_removeAllModels") + { + CatBoostLibraryHandlerFactory::instance().removeAllModels(); + + String res = "1"; + writeStringBinary(res, out); + } + else if (method == "catboost_GetTreeCount") + { + auto & read_buf = request.getStream(); + params.read(read_buf); + + if (!params.has("library_path")) + { + processError(response, "No 'library_path' in request URL"); + return; + } + + const String & library_path = params.get("library_path"); + + if (!params.has("model_path")) + { + processError(response, "No 'model_path' in request URL"); + return; + } + + const String & model_path = params.get("model_path"); + + auto catboost_handler = CatBoostLibraryHandlerFactory::instance().tryGetModel(model_path, library_path, /*create_if_not_found*/ true); + size_t tree_count = catboost_handler->getTreeCount(); + writeIntBinary(tree_count, out); + } + else if (method == "catboost_libEvaluate") + { + auto & read_buf = request.getStream(); + params.read(read_buf); + + if (!params.has("model_path")) + { + processError(response, "No 'model_path' in request URL"); + return; + } + + const String & model_path = params.get("model_path"); + + if (!params.has("data")) + { + processError(response, "No 'data' in request URL"); + return; + } + + const String & data = params.get("data"); + + ReadBufferFromString string_read_buf(data); + NativeReader deserializer(string_read_buf, /*server_revision*/ 0); + Block block_read = deserializer.read(); + + Columns col_ptrs = block_read.getColumns(); + ColumnRawPtrs col_raw_ptrs; + for (const auto & p : col_ptrs) + col_raw_ptrs.push_back(&*p); + + auto catboost_handler = CatBoostLibraryHandlerFactory::instance().tryGetModel(model_path, "DummyLibraryPath", /*create_if_not_found*/ false); + + if (!catboost_handler) + { + processError(response, "CatBoost library is not loaded for model '" + model_path + "'. Please try again."); + return; + } + + ColumnPtr res_col = catboost_handler->evaluate(col_raw_ptrs); + + DataTypePtr res_col_type = std::make_shared(); + String res_col_name = "res_col"; + + ColumnsWithTypeAndName res_cols_with_type_and_name = {{res_col, res_col_type, res_col_name}}; + + Block block_write(res_cols_with_type_and_name); + NativeWriter serializer{out, /*client_revision*/ 0, block_write}; + serializer.write(block_write); + } + else + { + processError(response, "Unknown library method '" + method + "'"); + LOG_ERROR(log, "Unknown library method: '{}'", method); + } + } + catch (...) + { + auto message = getCurrentExceptionMessage(true); + LOG_ERROR(log, "Failed to process request. Error: {}", message); + + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR, message); // can't call process_error, because of too soon response sending + try + { + writeStringBinary(message, out); + out.finalize(); + } + catch (...) + { + tryLogCurrentException(log); + } + } + + try + { + out.finalize(); + } + catch (...) + { + tryLogCurrentException(log); + } +} + + +CatBoostLibraryBridgeExistsHandler::CatBoostLibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_) + : WithContext(context_) + , keep_alive_timeout(keep_alive_timeout_) + , log(&Poco::Logger::get("CatBoostLibraryBridgeExistsHandler")) +{ +} + + +void CatBoostLibraryBridgeExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) +{ + try + { + LOG_TRACE(log, "Request URI: {}", request.getURI()); + HTMLForm params(getContext()->getSettingsRef(), request); + + String res = "1"; + + setResponseDefaultHeaders(response, keep_alive_timeout); + LOG_TRACE(log, "Sending ping response: {}", res); + response.sendBuffer(res.data(), res.size()); + } + catch (...) + { + tryLogCurrentException("PingHandler"); + } +} + } diff --git a/programs/library-bridge/LibraryBridgeHandlers.h b/programs/library-bridge/LibraryBridgeHandlers.h index b20f40616ce..16815e84723 100644 --- a/programs/library-bridge/LibraryBridgeHandlers.h +++ b/programs/library-bridge/LibraryBridgeHandlers.h @@ -1,9 +1,8 @@ #pragma once +#include #include #include -#include -#include "ExternalDictionaryLibraryHandler.h" namespace DB @@ -26,11 +25,12 @@ public: private: static constexpr inline auto FORMAT = "RowBinary"; + const size_t keep_alive_timeout; Poco::Logger * log; - size_t keep_alive_timeout; }; +// Handler for checking if the external dictionary library is loaded (used for handshake) class ExternalDictionaryLibraryBridgeExistsHandler : public HTTPRequestHandler, WithContext { public: @@ -43,4 +43,47 @@ private: Poco::Logger * log; }; + +/// Handler for requests to catboost library. The call protocol is as follows: +/// (1) Send a "catboost_GetTreeCount" request from the server to the bridge. It contains a library path (e.g /home/user/libcatboost.so) and +/// a model path (e.g. /home/user/model.bin). This loads the catboost library handler associated with the model path, then executes +/// GetTreeCount() on the library handler and sends the result back to the server. +/// (2) Send "catboost_Evaluate" from the server to the bridge. It contains a model path and the features to run the interference on. Step +/// (2) is called multiple times (once per chunk) by the server. +/// +/// We would ideally like to have steps (1) and (2) in one atomic handler but can't because the evaluation on the server side is divided +/// into two dependent phases: FunctionCatBoostEvaluate::getReturnTypeImpl() and ::executeImpl(). So the model may in principle be unloaded +/// from the library-bridge between steps (1) and (2). Step (2) checks if that is the case and fails gracefully. This is okay because that +/// situation considered exceptional and rare. +/// +/// An update of a model is performed by unloading it. The first call to "catboost_GetTreeCount" brings it into memory again. +/// +/// Further handlers are provided for unloading a specific model, for unloading all models or for retrieving information about the loaded +/// models for display in a system view. +class CatBoostLibraryBridgeRequestHandler : public HTTPRequestHandler, WithContext +{ +public: + CatBoostLibraryBridgeRequestHandler(size_t keep_alive_timeout_, ContextPtr context_); + + void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; + +private: + const size_t keep_alive_timeout; + Poco::Logger * log; +}; + + +// Handler for pinging the library-bridge for catboost access (used for handshake) +class CatBoostLibraryBridgeExistsHandler : public HTTPRequestHandler, WithContext +{ +public: + CatBoostLibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_); + + void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; + +private: + const size_t keep_alive_timeout; + Poco::Logger * log; +}; + } diff --git a/src/Common/SharedLibrary.cpp b/programs/library-bridge/SharedLibrary.cpp similarity index 95% rename from src/Common/SharedLibrary.cpp rename to programs/library-bridge/SharedLibrary.cpp index 6104c96676a..d70709474b5 100644 --- a/src/Common/SharedLibrary.cpp +++ b/programs/library-bridge/SharedLibrary.cpp @@ -1,8 +1,7 @@ #include "SharedLibrary.h" #include -#include #include -#include "Exception.h" +#include namespace DB diff --git a/src/Common/SharedLibrary.h b/programs/library-bridge/SharedLibrary.h similarity index 100% rename from src/Common/SharedLibrary.h rename to programs/library-bridge/SharedLibrary.h diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index ce31600642a..2b9d819f5eb 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -227,6 +227,8 @@ void LocalServer::cleanup() global_context.reset(); } + /// thread status should be destructed before shared context because it relies on process list. + status.reset(); // Delete the temporary directory if needed. @@ -366,7 +368,7 @@ int LocalServer::main(const std::vector & /*args*/) try { UseSSL use_ssl; - ThreadStatus thread_status; + thread_status.emplace(); StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true)); @@ -620,9 +622,13 @@ void LocalServer::processConfig() attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); - loadMetadata(global_context); startupSystemTables(); - DatabaseCatalog::instance().loadDatabases(); + + if (!config().has("only-system-tables")) + { + loadMetadata(global_context); + DatabaseCatalog::instance().loadDatabases(); + } LOG_DEBUG(log, "Loaded metadata."); } @@ -713,6 +719,7 @@ void LocalServer::addOptions(OptionsDescription & options_description) ("no-system-tables", "do not attach system tables (better startup time)") ("path", po::value(), "Storage path") + ("only-system-tables", "attach only system tables from specified path") ("top_level_domains_path", po::value(), "Path to lists with custom TLDs") ; } @@ -741,6 +748,8 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp config().setString("table-structure", options["structure"].as()); if (options.count("no-system-tables")) config().setBool("no-system-tables", true); + if (options.count("only-system-tables")) + config().setBool("only-system-tables", true); if (options.count("input-format")) config().setString("table-data-format", options["input-format"].as()); diff --git a/programs/local/clickhouse-local.cpp b/programs/local/clickhouse-local.cpp index 5cc98ab6067..4b17e89496d 100644 --- a/programs/local/clickhouse-local.cpp +++ b/programs/local/clickhouse-local.cpp @@ -1,2 +1 @@ int mainEntryClickHouseLocal(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseLocal(argc_, argv_); } diff --git a/programs/main.cpp b/programs/main.cpp index fef0ad688e2..9b0e890cd76 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -402,6 +402,36 @@ void checkHarmfulEnvironmentVariables(char ** argv) } +/// Don't allow dlopen in the main ClickHouse binary, because it is harmful and insecure. +/// We don't use it. But it can be used by some libraries for implementation of "plugins". +/// We absolutely discourage the ancient technique of loading +/// 3rd-party uncontrolled dangerous libraries into the process address space, +/// because it is insane. + +extern "C" +{ + void * dlopen(const char *, int) + { + return nullptr; + } + + void * dlmopen(long, const char *, int) // NOLINT + { + return nullptr; + } + + int dlclose(void *) + { + return 0; + } + + const char * dlerror() + { + return "ClickHouse does not allow dynamic library loading"; + } +} + + /// This allows to implement assert to forbid initialization of a class in static constructors. /// Usage: /// @@ -422,6 +452,7 @@ int main(int argc_, char ** argv_) /// PHDR cache is required for query profiler to work reliably /// It also speed up exception handling, but exceptions from dynamically loaded libraries (dlopen) /// will work only after additional call of this function. + /// Note: we forbid dlopen in our code. updatePHDRCache(); #ifndef DISABLE_HARMFUL_ENV_VAR_CHECK diff --git a/programs/obfuscator/clickhouse-obfuscator.cpp b/programs/obfuscator/clickhouse-obfuscator.cpp index e57fa6d1b54..1f494223150 100644 --- a/programs/obfuscator/clickhouse-obfuscator.cpp +++ b/programs/obfuscator/clickhouse-obfuscator.cpp @@ -1,3 +1 @@ int mainEntryClickHouseObfuscator(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseObfuscator(argc_, argv_); } - diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 414766ee42a..8a0ce75ca70 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -51,7 +51,6 @@ #include #include #include -#include #include #include #include @@ -551,8 +550,9 @@ static void sanityChecks(Server & server) try { const char * filename = "/sys/devices/system/clocksource/clocksource0/current_clocksource"; - if (readString(filename).find("tsc") == std::string::npos) - server.context()->addWarningMessage("Linux is not using a fast TSC clock source. Performance can be degraded. Check " + String(filename)); + String clocksource = readString(filename); + if (clocksource.find("tsc") == std::string::npos && clocksource.find("kvm-clock") == std::string::npos) + server.context()->addWarningMessage("Linux is not using a fast clock source. Performance can be degraded. Check " + String(filename)); } catch (...) { @@ -1157,7 +1157,6 @@ int Server::main(const std::vector & /*args*/) global_context->setExternalAuthenticatorsConfig(*config); global_context->loadOrReloadDictionaries(*config); - global_context->loadOrReloadModels(*config); global_context->loadOrReloadUserDefinedExecutableFunctions(*config); global_context->setRemoteHostFilter(*config); @@ -1382,7 +1381,7 @@ int Server::main(const std::vector & /*args*/) global_context->setConfigReloadCallback([&]() { main_config_reloader->reload(); - access_control.reload(); + access_control.reload(AccessControl::ReloadMode::USERS_CONFIG_ONLY); }); /// Limit on total number of concurrently executed queries. @@ -1475,6 +1474,23 @@ int Server::main(const std::vector & /*args*/) /// try set up encryption. There are some errors in config, error will be printed and server wouldn't start. CompressionCodecEncrypted::Configuration::instance().load(config(), "encryption_codecs"); + std::unique_ptr dns_cache_updater; + if (config().has("disable_internal_dns_cache") && config().getInt("disable_internal_dns_cache")) + { + /// Disable DNS caching at all + DNSResolver::instance().setDisableCacheFlag(); + LOG_DEBUG(log, "DNS caching disabled"); + } + else + { + /// Initialize a watcher periodically updating DNS cache + dns_cache_updater = std::make_unique( + global_context, config().getInt("dns_cache_update_period", 15), config().getUInt("dns_max_consecutive_failures", 5)); + } + + if (dns_cache_updater) + dns_cache_updater->start(); + SCOPE_EXIT({ /// Stop reloading of the main config. This must be done before `global_context->shutdown()` because /// otherwise the reloading may pass a changed config to some destroyed parts of ContextSharedPart. @@ -1628,20 +1644,6 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); - std::unique_ptr dns_cache_updater; - if (config().has("disable_internal_dns_cache") && config().getInt("disable_internal_dns_cache")) - { - /// Disable DNS caching at all - DNSResolver::instance().setDisableCacheFlag(); - LOG_DEBUG(log, "DNS caching disabled"); - } - else - { - /// Initialize a watcher periodically updating DNS cache - dns_cache_updater = std::make_unique( - global_context, config().getInt("dns_cache_update_period", 15), config().getUInt("dns_max_consecutive_failures", 5)); - } - #if defined(OS_LINUX) auto tasks_stats_provider = TasksStatsCounters::findBestAvailableProvider(); if (tasks_stats_provider == TasksStatsCounters::MetricsProvider::None) @@ -1706,8 +1708,6 @@ int Server::main(const std::vector & /*args*/) main_config_reloader->start(); access_control.startPeriodicReloading(); - if (dns_cache_updater) - dns_cache_updater->start(); { LOG_INFO(log, "Available RAM: {}; physical cores: {}; logical cores: {}.", @@ -1738,17 +1738,6 @@ int Server::main(const std::vector & /*args*/) throw; } - /// try to load models immediately, throw on error and die - try - { - global_context->loadOrReloadModels(config()); - } - catch (...) - { - tryLogCurrentException(log, "Caught exception while loading dictionaries."); - throw; - } - /// try to load user defined executable functions, throw on error and die try { diff --git a/programs/server/clickhouse-server.cpp b/programs/server/clickhouse-server.cpp index f49dc21d9fe..a7a3d8f00e3 100644 --- a/programs/server/clickhouse-server.cpp +++ b/programs/server/clickhouse-server.cpp @@ -1,24 +1 @@ -#include - -#include - - int mainEntryClickHouseServer(int argc, char ** argv); - -/** - * This is the entry-point for the split build server. The initialization - * is copied from single-binary entry point in main.cpp. - */ -int main(int argc_, char ** argv_) -{ - /// Reset new handler to default (that throws std::bad_alloc) - /// It is needed because LLVM library clobbers it. - std::set_new_handler(nullptr); - - /// PHDR cache is required for query profiler to work reliably - /// It also speed up exception handling, but exceptions from dynamically loaded libraries (dlopen) - /// will work only after additional call of this function. - updatePHDRCache(); - - return mainEntryClickHouseServer(argc_, argv_); -} diff --git a/programs/server/config.xml b/programs/server/config.xml index a1e139d9e76..448203a72ec 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -65,7 +65,7 @@ For example, as below: {"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"} To enable JSON logging support, please uncomment the entire tag below. - + a) You can modify key names by changing values under tag values inside tag. For example, to change DATE_TIME to MY_DATE_TIME, you can do like: MY_DATE_TIME @@ -661,6 +661,13 @@ executed by any user. You can change this behaviour by setting this to true. If it's set to true then this query requires "GRANT SELECT ON information_schema." just like as for ordinary tables. --> false + + + false diff --git a/programs/static-files-disk-uploader/clickhouse-static-files-disk-uploader.cpp b/programs/static-files-disk-uploader/clickhouse-static-files-disk-uploader.cpp index 063604b10b1..cf271f16a76 100644 --- a/programs/static-files-disk-uploader/clickhouse-static-files-disk-uploader.cpp +++ b/programs/static-files-disk-uploader/clickhouse-static-files-disk-uploader.cpp @@ -1,2 +1 @@ int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseStaticFilesDiskUploader(argc_, argv_); } diff --git a/programs/su/clickhouse-su.cpp b/programs/su/clickhouse-su.cpp index bb0967ca271..9459ccad0e8 100644 --- a/programs/su/clickhouse-su.cpp +++ b/programs/su/clickhouse-su.cpp @@ -1,2 +1 @@ int mainEntryClickHouseSU(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseSU(argc_, argv_); } diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 89292fe9272..e7dfdc976b9 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -171,6 +171,7 @@ void AccessControl::setUpFromMainConfig(const Poco::Util::AbstractConfiguration setOnClusterQueriesRequireClusterGrant(config_.getBool("access_control_improvements.on_cluster_queries_require_cluster_grant", false)); setSelectFromSystemDatabaseRequiresGrant(config_.getBool("access_control_improvements.select_from_system_db_requires_grant", false)); setSelectFromInformationSchemaRequiresGrant(config_.getBool("access_control_improvements.select_from_information_schema_requires_grant", false)); + setSettingsConstraintsReplacePrevious(config_.getBool("access_control_improvements.settings_constraints_replace_previous", false)); addStoragesFromMainConfig(config_, config_path_, get_zookeeper_function_); } @@ -390,9 +391,9 @@ void AccessControl::addStoragesFromMainConfig( } -void AccessControl::reload() +void AccessControl::reload(ReloadMode reload_mode) { - MultipleAccessStorage::reload(); + MultipleAccessStorage::reload(reload_mode); changes_notifier->sendNotifications(); } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index ab9cdba9ad1..bf42941f501 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -99,8 +99,8 @@ public: const String & config_path, const zkutil::GetZooKeeper & get_zookeeper_function); - /// Reloads and updates entities in this storage. This function is used to implement SYSTEM RELOAD CONFIG. - void reload() override; + /// Reloads and updates all access entities. + void reload(ReloadMode reload_mode) override; using OnChangedHandler = std::function; @@ -158,6 +158,9 @@ public: void setSelectFromInformationSchemaRequiresGrant(bool enable) { select_from_information_schema_requires_grant = enable; } bool doesSelectFromInformationSchemaRequireGrant() const { return select_from_information_schema_requires_grant; } + void setSettingsConstraintsReplacePrevious(bool enable) { settings_constraints_replace_previous = enable; } + bool doesSettingsConstraintsReplacePrevious() const { return settings_constraints_replace_previous; } + std::shared_ptr getContextAccess( const UUID & user_id, const std::vector & current_roles, @@ -223,6 +226,7 @@ private: std::atomic_bool on_cluster_queries_require_cluster_grant = false; std::atomic_bool select_from_system_db_requires_grant = false; std::atomic_bool select_from_information_schema_requires_grant = false; + std::atomic_bool settings_constraints_replace_previous = false; }; } diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index e8d3b453188..de44874238e 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -143,6 +143,7 @@ enum class AccessType M(SYSTEM_DROP_SCHEMA_CACHE, "SYSTEM DROP SCHEMA CACHE, DROP SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_CACHE, "DROP CACHE", GROUP, SYSTEM) \ M(SYSTEM_RELOAD_CONFIG, "RELOAD CONFIG", GLOBAL, SYSTEM_RELOAD) \ + M(SYSTEM_RELOAD_USERS, "RELOAD USERS", GLOBAL, SYSTEM_RELOAD) \ M(SYSTEM_RELOAD_SYMBOLS, "RELOAD SYMBOLS", GLOBAL, SYSTEM_RELOAD) \ M(SYSTEM_RELOAD_DICTIONARY, "SYSTEM RELOAD DICTIONARIES, RELOAD DICTIONARY, RELOAD DICTIONARIES", GLOBAL, SYSTEM_RELOAD) \ M(SYSTEM_RELOAD_MODEL, "SYSTEM RELOAD MODELS, RELOAD MODEL, RELOAD MODELS", GLOBAL, SYSTEM_RELOAD) \ diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 0cbe420f345..ff51ae1cfd4 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -182,8 +183,8 @@ DiskAccessStorage::DiskAccessStorage(const String & storage_name_, const String if (should_rebuild_lists) { - rebuildLists(); - writeLists(); + LOG_WARNING(getLogger(), "Recovering lists in directory {}", directory_path); + reloadAllAndRebuildLists(); } } @@ -224,63 +225,57 @@ bool DiskAccessStorage::isPathEqual(const String & directory_path_) const } -void DiskAccessStorage::clear() -{ - entries_by_id.clear(); - for (auto type : collections::range(AccessEntityType::MAX)) - entries_by_name_and_type[static_cast(type)].clear(); -} - - bool DiskAccessStorage::readLists() { - clear(); + std::vector> ids_names_types; - bool ok = true; for (auto type : collections::range(AccessEntityType::MAX)) { - auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; auto file_path = getListFilePath(directory_path, type); if (!std::filesystem::exists(file_path)) { LOG_WARNING(getLogger(), "File {} doesn't exist", file_path); - ok = false; - break; + return false; } try { for (const auto & [id, name] : readListFile(file_path)) - { - auto & entry = entries_by_id[id]; - entry.id = id; - entry.type = type; - entry.name = name; - entries_by_name[entry.name] = &entry; - } + ids_names_types.emplace_back(id, name, type); } catch (...) { tryLogCurrentException(getLogger(), "Could not read " + file_path); - ok = false; - break; + return false; } } - if (!ok) - clear(); - return ok; + entries_by_id.clear(); + for (auto type : collections::range(AccessEntityType::MAX)) + entries_by_name_and_type[static_cast(type)].clear(); + + for (auto & [id, name, type] : ids_names_types) + { + auto & entry = entries_by_id[id]; + entry.id = id; + entry.type = type; + entry.name = std::move(name); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + entries_by_name[entry.name] = &entry; + } + + return true; } -bool DiskAccessStorage::writeLists() +void DiskAccessStorage::writeLists() { if (failed_to_write_lists) - return false; /// We don't try to write list files after the first fail. - /// The next restart of the server will invoke rebuilding of the list files. + return; /// We don't try to write list files after the first fail. + /// The next restart of the server will invoke rebuilding of the list files. if (types_of_lists_to_write.empty()) - return true; + return; for (const auto & type : types_of_lists_to_write) { @@ -299,14 +294,13 @@ bool DiskAccessStorage::writeLists() tryLogCurrentException(getLogger(), "Could not write " + file_path); failed_to_write_lists = true; types_of_lists_to_write.clear(); - return false; + return; } } /// The list files was successfully written, we don't need the 'need_rebuild_lists.mark' file any longer. std::filesystem::remove(getNeedRebuildListsMarkFilePath(directory_path)); types_of_lists_to_write.clear(); - return true; } @@ -364,10 +358,9 @@ void DiskAccessStorage::stopListsWritingThread() /// Reads and parses all the ".sql" files from a specified directory /// and then saves the files "users.list", "roles.list", etc. to the same directory. -bool DiskAccessStorage::rebuildLists() +void DiskAccessStorage::reloadAllAndRebuildLists() { - LOG_WARNING(getLogger(), "Recovering lists in directory {}", directory_path); - clear(); + std::vector> all_entities; for (const auto & directory_entry : std::filesystem::directory_iterator(directory_path)) { @@ -386,21 +379,55 @@ bool DiskAccessStorage::rebuildLists() if (!entity) continue; - const String & name = entity->getName(); - auto type = entity->getType(); - auto & entry = entries_by_id[id]; - entry.id = id; - entry.type = type; - entry.name = name; - entry.entity = entity; - auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; - entries_by_name[entry.name] = &entry; + all_entities.emplace_back(id, entity); } + setAllInMemory(all_entities); + for (auto type : collections::range(AccessEntityType::MAX)) types_of_lists_to_write.insert(type); - return true; + failed_to_write_lists = false; /// Try again writing lists. + writeLists(); +} + +void DiskAccessStorage::setAllInMemory(const std::vector> & all_entities) +{ + /// Remove conflicting entities from the specified list. + auto entities_without_conflicts = all_entities; + clearConflictsInEntitiesList(entities_without_conflicts, getLogger()); + + /// Remove entities which are not used anymore. + boost::container::flat_set ids_to_keep; + ids_to_keep.reserve(entities_without_conflicts.size()); + for (const auto & [id, _] : entities_without_conflicts) + ids_to_keep.insert(id); + removeAllExceptInMemory(ids_to_keep); + + /// Insert or update entities. + for (const auto & [id, entity] : entities_without_conflicts) + insertNoLock(id, entity, /* replace_if_exists = */ true, /* throw_if_exists = */ false, /* write_on_disk= */ false); +} + +void DiskAccessStorage::removeAllExceptInMemory(const boost::container::flat_set & ids_to_keep) +{ + for (auto it = entries_by_id.begin(); it != entries_by_id.end();) + { + const auto & id = it->first; + ++it; /// We must go to the next element in the map `entries_by_id` here because otherwise removeNoLock() can invalidate our iterator. + if (!ids_to_keep.contains(id)) + removeNoLock(id, /* throw_if_not_exists */ true, /* write_on_disk= */ false); + } +} + + +void DiskAccessStorage::reload(ReloadMode reload_mode) +{ + if (reload_mode != ReloadMode::ALL) + return; + + std::lock_guard lock{mutex}; + reloadAllAndRebuildLists(); } @@ -471,21 +498,21 @@ std::optional> DiskAccessStorage::readNameWi std::optional DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) { UUID id = generateRandomID(); - if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists)) + if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists, /* write_on_disk= */ true)) return id; return std::nullopt; } -bool DiskAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) +bool DiskAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) { std::lock_guard lock{mutex}; - return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists); + return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, write_on_disk); } -bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) +bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) { const String & name = new_entity->getName(); AccessEntityType type = new_entity->getType(); @@ -497,6 +524,9 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; auto it_by_name = entries_by_name.find(name); bool name_collision = (it_by_name != entries_by_name.end()); + UUID id_by_name; + if (name_collision) + id_by_name = it_by_name->second->id; if (name_collision && !replace_if_exists) { @@ -507,19 +537,57 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne } auto it_by_id = entries_by_id.find(id); - if (it_by_id != entries_by_id.end()) + bool id_collision = (it_by_id != entries_by_id.end()); + if (id_collision && !replace_if_exists) { - const auto & existing_entry = it_by_id->second; - throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); + if (throw_if_exists) + { + const auto & existing_entry = it_by_id->second; + throwIDCollisionCannotInsert(id, type, name, existing_entry.type, existing_entry.name); + } + else + return false; } - scheduleWriteLists(type); - writeAccessEntityToDisk(id, *new_entity); + if (write_on_disk) + scheduleWriteLists(type); - if (name_collision && replace_if_exists) - removeNoLock(it_by_name->second->id, /* throw_if_not_exists = */ false); + /// Remove collisions if necessary. + if (name_collision && (id_by_name != id)) + { + assert(replace_if_exists); + removeNoLock(id_by_name, /* throw_if_not_exists= */ false, write_on_disk); + } + + if (id_collision) + { + assert(replace_if_exists); + auto & existing_entry = it_by_id->second; + if (existing_entry.type == new_entity->getType()) + { + if (!existing_entry.entity || (*existing_entry.entity != *new_entity)) + { + if (write_on_disk) + writeAccessEntityToDisk(id, *new_entity); + if (existing_entry.name != new_entity->getName()) + { + entries_by_name.erase(existing_entry.name); + [[maybe_unused]] bool inserted = entries_by_name.emplace(new_entity->getName(), &existing_entry).second; + assert(inserted); + } + existing_entry.entity = new_entity; + changes_notifier.onEntityUpdated(id, new_entity); + } + return true; + } + + removeNoLock(id, /* throw_if_not_exists= */ false, write_on_disk); + } /// Do insertion. + if (write_on_disk) + writeAccessEntityToDisk(id, *new_entity); + auto & entry = entries_by_id[id]; entry.id = id; entry.type = type; @@ -535,11 +603,11 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne bool DiskAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists) { std::lock_guard lock{mutex}; - return removeNoLock(id, throw_if_not_exists); + return removeNoLock(id, throw_if_not_exists, /* write_on_disk= */ true); } -bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists) +bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk) { auto it = entries_by_id.find(id); if (it == entries_by_id.end()) @@ -556,8 +624,11 @@ bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists) if (readonly) throwReadonlyCannotRemove(type, entry.name); - scheduleWriteLists(type); - deleteAccessEntityOnDisk(id); + if (write_on_disk) + { + scheduleWriteLists(type); + deleteAccessEntityOnDisk(id); + } /// Do removing. UUID removed_id = id; @@ -573,11 +644,11 @@ bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists) bool DiskAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) { std::lock_guard lock{mutex}; - return updateNoLock(id, update_func, throw_if_not_exists); + return updateNoLock(id, update_func, throw_if_not_exists, /* write_on_disk= */ true); } -bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) +bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, bool write_on_disk) { auto it = entries_by_id.find(id); if (it == entries_by_id.end()) @@ -613,10 +684,13 @@ bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ { if (entries_by_name.contains(new_name)) throwNameCollisionCannotRename(type, old_name, new_name); - scheduleWriteLists(type); + if (write_on_disk) + scheduleWriteLists(type); } - writeAccessEntityToDisk(id, *new_entity); + if (write_on_disk) + writeAccessEntityToDisk(id, *new_entity); + entry.entity = new_entity; if (name_changed) @@ -668,7 +742,7 @@ void DiskAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists] { for (const auto & [id, entity] : entities) - insertWithID(id, entity, replace_if_exists, throw_if_exists); + insertWithID(id, entity, replace_if_exists, throw_if_exists, /* write_on_disk= */ true); }); } diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h index d3bd61ff353..b1ef1d10ba7 100644 --- a/src/Access/DiskAccessStorage.h +++ b/src/Access/DiskAccessStorage.h @@ -27,6 +27,8 @@ public: void setReadOnly(bool readonly_) { readonly = readonly_; } bool isReadOnly() const override { return readonly; } + void reload(ReloadMode reload_mode) override; + bool exists(const UUID & id) const override; bool isBackupAllowed() const override { return backup_allowed; } @@ -41,19 +43,20 @@ private: bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; - void clear(); - bool readLists(); - bool writeLists(); - void scheduleWriteLists(AccessEntityType type); - bool rebuildLists(); + bool readLists() TSA_REQUIRES(mutex); + void writeLists() TSA_REQUIRES(mutex); + void scheduleWriteLists(AccessEntityType type) TSA_REQUIRES(mutex); + void reloadAllAndRebuildLists() TSA_REQUIRES(mutex); + void setAllInMemory(const std::vector> & all_entities) TSA_REQUIRES(mutex); + void removeAllExceptInMemory(const boost::container::flat_set & ids_to_keep) TSA_REQUIRES(mutex); - void listsWritingThreadFunc(); + void listsWritingThreadFunc() TSA_NO_THREAD_SAFETY_ANALYSIS; void stopListsWritingThread(); - bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); - bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); - bool removeNoLock(const UUID & id, bool throw_if_not_exists); - bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); + bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk); + bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) TSA_REQUIRES(mutex); + bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex); + bool removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex); AccessEntityPtr readAccessEntityFromDisk(const UUID & id) const; void writeAccessEntityToDisk(const UUID & id, const IAccessEntity & entity) const; @@ -69,13 +72,22 @@ private: }; String directory_path; - std::unordered_map entries_by_id; - std::unordered_map entries_by_name_and_type[static_cast(AccessEntityType::MAX)]; - boost::container::flat_set types_of_lists_to_write; - bool failed_to_write_lists = false; /// Whether writing of the list files has been failed since the recent restart of the server. - ThreadFromGlobalPool lists_writing_thread; /// List files are written in a separate thread. - std::condition_variable lists_writing_thread_should_exit; /// Signals `lists_writing_thread` to exit. + + std::unordered_map entries_by_id TSA_GUARDED_BY(mutex); + std::unordered_map entries_by_name_and_type[static_cast(AccessEntityType::MAX)] TSA_GUARDED_BY(mutex); + boost::container::flat_set types_of_lists_to_write TSA_GUARDED_BY(mutex); + + /// Whether writing of the list files has been failed since the recent restart of the server. + bool failed_to_write_lists TSA_GUARDED_BY(mutex) = false; + + /// List files are written in a separate thread. + ThreadFromGlobalPool lists_writing_thread; + + /// Signals `lists_writing_thread` to exit. + std::condition_variable lists_writing_thread_should_exit; + bool lists_writing_thread_is_waiting = false; + AccessChangesNotifier & changes_notifier; std::atomic readonly; std::atomic backup_allowed; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 2be99dfb38b..f562a6ebeec 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -562,6 +563,62 @@ UUID IAccessStorage::generateRandomID() } +void IAccessStorage::clearConflictsInEntitiesList(std::vector> & entities, const Poco::Logger * log_) +{ + std::unordered_map positions_by_id; + std::unordered_map positions_by_type_and_name[static_cast(AccessEntityType::MAX)]; + std::vector positions_to_remove; + + for (size_t pos = 0; pos != entities.size(); ++pos) + { + const auto & [id, entity] = entities[pos]; + + if (auto it = positions_by_id.find(id); it == positions_by_id.end()) + { + positions_by_id[id] = pos; + } + else if (it->second != pos) + { + /// Conflict: same ID is used for multiple entities. We will ignore them. + positions_to_remove.emplace_back(pos); + positions_to_remove.emplace_back(it->second); + } + + std::string_view entity_name = entity->getName(); + auto & positions_by_name = positions_by_type_and_name[static_cast(entity->getType())]; + if (auto it = positions_by_name.find(entity_name); it == positions_by_name.end()) + { + positions_by_name[entity_name] = pos; + } + else if (it->second != pos) + { + /// Conflict: same name and type are used for multiple entities. We will ignore them. + positions_to_remove.emplace_back(pos); + positions_to_remove.emplace_back(it->second); + } + } + + if (positions_to_remove.empty()) + return; + + std::sort(positions_to_remove.begin(), positions_to_remove.end()); + positions_to_remove.erase(std::unique(positions_to_remove.begin(), positions_to_remove.end()), positions_to_remove.end()); + + for (size_t pos : positions_to_remove) + { + LOG_WARNING( + log_, + "Skipping {} (id={}) due to conflicts with other access entities", + entities[pos].second->formatTypeWithName(), + toString(entities[pos].first)); + } + + /// Remove conflicting entities. + for (size_t pos : positions_to_remove | boost::adaptors::reversed) /// Must remove in reversive order. + entities.erase(entities.begin() + pos); +} + + Poco::Logger * IAccessStorage::getLogger() const { Poco::Logger * ptr = log.load(); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 394d3ed6358..aa3947201e7 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -42,15 +42,26 @@ public: /// Returns true if this entity is readonly. virtual bool isReadOnly(const UUID &) const { return isReadOnly(); } - /// Reloads and updates entities in this storage. This function is used to implement SYSTEM RELOAD CONFIG. - virtual void reload() {} - - /// Starts periodic reloading and update of entities in this storage. + /// Starts periodic reloading and updating of entities in this storage. virtual void startPeriodicReloading() {} - /// Stops periodic reloading and update of entities in this storage. + /// Stops periodic reloading and updating of entities in this storage. virtual void stopPeriodicReloading() {} + enum class ReloadMode + { + /// Try to reload all access storages (including users.xml, local(disk) access storage, replicated(in zk) access storage. + /// This mode is invoked by the SYSTEM RELOAD USERS command. + ALL, + + /// Only reloads users.xml + /// This mode is invoked by the SYSTEM RELOAD CONFIG command. + USERS_CONFIG_ONLY, + }; + + /// Makes this storage to reload and update access entities right now. + virtual void reload(ReloadMode /* reload_mode */) {} + /// Returns the identifiers of all the entities of a specified type contained in the storage. std::vector findAll(AccessEntityType type) const; @@ -177,6 +188,7 @@ protected: static UUID generateRandomID(); Poco::Logger * getLogger() const; static String formatEntityTypeWithName(AccessEntityType type, const String & name) { return AccessEntityTypeInfo::get(type).formatEntityNameWithType(name); } + static void clearConflictsInEntitiesList(std::vector> & entities, const Poco::Logger * log_); [[noreturn]] void throwNotFound(const UUID & id) const; [[noreturn]] void throwNotFound(AccessEntityType type, const String & name) const; [[noreturn]] static void throwBadCast(const UUID & id, AccessEntityType type, const String & name, AccessEntityType required_type); diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 60669532e25..8fcca235ee8 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -5,7 +5,6 @@ #include #include #include -#include namespace DB @@ -90,6 +89,9 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; auto it_by_name = entries_by_name.find(name); bool name_collision = (it_by_name != entries_by_name.end()); + UUID id_by_name; + if (name_collision) + id_by_name = it_by_name->second->id; if (name_collision && !replace_if_exists) { @@ -100,16 +102,43 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & } auto it_by_id = entries_by_id.find(id); - if (it_by_id != entries_by_id.end()) + bool id_collision = (it_by_id != entries_by_id.end()); + if (id_collision && !replace_if_exists) { const auto & existing_entry = it_by_id->second; - throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); + if (throw_if_exists) + throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); + else + return false; } - if (name_collision && replace_if_exists) + /// Remove collisions if necessary. + if (name_collision && (id_by_name != id)) { - const auto & existing_entry = *(it_by_name->second); - removeNoLock(existing_entry.id, /* throw_if_not_exists = */ false); + assert(replace_if_exists); + removeNoLock(id_by_name, /* throw_if_not_exists= */ true); + } + + if (id_collision) + { + assert(replace_if_exists); + auto & existing_entry = it_by_id->second; + if (existing_entry.entity->getType() == new_entity->getType()) + { + if (*existing_entry.entity != *new_entity) + { + if (existing_entry.entity->getName() != new_entity->getName()) + { + entries_by_name.erase(existing_entry.entity->getName()); + [[maybe_unused]] bool inserted = entries_by_name.emplace(new_entity->getName(), &existing_entry).second; + assert(inserted); + } + existing_entry.entity = new_entity; + changes_notifier.onEntityUpdated(id, new_entity); + } + return true; + } + removeNoLock(id, /* throw_if_not_exists= */ true); } /// Do insertion. @@ -201,6 +230,29 @@ bool MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & updat } +void MemoryAccessStorage::removeAllExcept(const std::vector & ids_to_keep) +{ + std::lock_guard lock{mutex}; + removeAllExceptNoLock(ids_to_keep); +} + +void MemoryAccessStorage::removeAllExceptNoLock(const std::vector & ids_to_keep) +{ + removeAllExceptNoLock(boost::container::flat_set{ids_to_keep.begin(), ids_to_keep.end()}); +} + +void MemoryAccessStorage::removeAllExceptNoLock(const boost::container::flat_set & ids_to_keep) +{ + for (auto it = entries_by_id.begin(); it != entries_by_id.end();) + { + const auto & id = it->first; + ++it; /// We must go to the next element in the map `entries_by_id` here because otherwise removeNoLock() can invalidate our iterator. + if (!ids_to_keep.contains(id)) + removeNoLock(id, /* throw_if_not_exists */ true); + } +} + + void MemoryAccessStorage::setAll(const std::vector & all_entities) { std::vector> entities_with_ids; @@ -215,61 +267,20 @@ void MemoryAccessStorage::setAll(const std::vector not_used_ids; - std::vector conflicting_ids; + /// Remove conflicting entities from the specified list. + auto entities_without_conflicts = all_entities; + clearConflictsInEntitiesList(entities_without_conflicts, getLogger()); - /// Get the list of currently used IDs. Later we will remove those of them which are not used anymore. - for (const auto & id : entries_by_id | boost::adaptors::map_keys) - not_used_ids.emplace(id); - - /// Get the list of conflicting IDs and update the list of currently used ones. - for (const auto & [id, entity] : all_entities) - { - auto it = entries_by_id.find(id); - if (it != entries_by_id.end()) - { - not_used_ids.erase(id); /// ID is used. - - Entry & entry = it->second; - if (entry.entity->getType() != entity->getType()) - conflicting_ids.emplace_back(id); /// Conflict: same ID, different type. - } - - const auto & entries_by_name = entries_by_name_and_type[static_cast(entity->getType())]; - auto it2 = entries_by_name.find(entity->getName()); - if (it2 != entries_by_name.end()) - { - Entry & entry = *(it2->second); - if (entry.id != id) - conflicting_ids.emplace_back(entry.id); /// Conflict: same name and type, different ID. - } - } - - /// Remove entities which are not used anymore and which are in conflict with new entities. - boost::container::flat_set ids_to_remove = std::move(not_used_ids); - boost::range::copy(conflicting_ids, std::inserter(ids_to_remove, ids_to_remove.end())); - for (const auto & id : ids_to_remove) - removeNoLock(id, /* throw_if_not_exists = */ false); + /// Remove entities which are not used anymore. + boost::container::flat_set ids_to_keep; + ids_to_keep.reserve(entities_without_conflicts.size()); + for (const auto & [id, _] : entities_without_conflicts) + ids_to_keep.insert(id); + removeAllExceptNoLock(ids_to_keep); /// Insert or update entities. - for (const auto & [id, entity] : all_entities) - { - auto it = entries_by_id.find(id); - if (it != entries_by_id.end()) - { - if (*(it->second.entity) != *entity) - { - const AccessEntityPtr & changed_entity = entity; - updateNoLock(id, - [&changed_entity](const AccessEntityPtr &) { return changed_entity; }, - /* throw_if_not_exists = */ true); - } - } - else - { - insertNoLock(id, entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true); - } - } + for (const auto & [id, entity] : entities_without_conflicts) + insertNoLock(id, entity, /* replace_if_exists = */ true, /* throw_if_exists = */ false); } diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h index 5c8d33ed443..b63132147da 100644 --- a/src/Access/MemoryAccessStorage.h +++ b/src/Access/MemoryAccessStorage.h @@ -22,6 +22,15 @@ public: const char * getStorageType() const override { return STORAGE_TYPE; } + /// Inserts an entity with a specified ID. + /// If `replace_if_exists == true` it can replace an existing entry with such ID and also remove an existing entry + /// with such name & type. + bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); + + /// Removes all entities except the specified list `ids_to_keep`. + /// The function skips IDs not contained in the storage. + void removeAllExcept(const std::vector & ids_to_keep); + /// Sets all entities at once. void setAll(const std::vector & all_entities); void setAll(const std::vector> & all_entities); @@ -39,11 +48,13 @@ private: bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; - bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) TSA_REQUIRES(mutex); bool removeNoLock(const UUID & id, bool throw_if_not_exists) TSA_REQUIRES(mutex); bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) TSA_REQUIRES(mutex); + void removeAllExceptNoLock(const std::vector & ids_to_keep) TSA_REQUIRES(mutex); + void removeAllExceptNoLock(const boost::container::flat_set & ids_to_keep) TSA_REQUIRES(mutex); + struct Entry { UUID id; @@ -54,6 +65,6 @@ private: std::unordered_map entries_by_id TSA_GUARDED_BY(mutex); /// We want to search entries both by ID and by the pair of name and type. std::unordered_map entries_by_name_and_type[static_cast(AccessEntityType::MAX)] TSA_GUARDED_BY(mutex); AccessChangesNotifier & changes_notifier; - bool backup_allowed = false; + const bool backup_allowed = false; }; } diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index e7151cc7b4b..f910c99d6b3 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -223,13 +223,6 @@ bool MultipleAccessStorage::isReadOnly(const UUID & id) const } -void MultipleAccessStorage::reload() -{ - auto storages = getStoragesInternal(); - for (const auto & storage : *storages) - storage->reload(); -} - void MultipleAccessStorage::startPeriodicReloading() { auto storages = getStoragesInternal(); @@ -244,6 +237,13 @@ void MultipleAccessStorage::stopPeriodicReloading() storage->stopPeriodicReloading(); } +void MultipleAccessStorage::reload(ReloadMode reload_mode) +{ + auto storages = getStoragesInternal(); + for (const auto & storage : *storages) + storage->reload(reload_mode); +} + std::optional MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) { diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index bc9a4705785..6a0c1bdfc02 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -25,9 +25,9 @@ public: bool isReadOnly() const override; bool isReadOnly(const UUID & id) const override; - void reload() override; void startPeriodicReloading() override; void stopPeriodicReloading() override; + void reload(ReloadMode reload_mode) override; void setStorages(const std::vector & storages); void addStorage(const StoragePtr & new_storage); diff --git a/src/Access/QuotaCache.cpp b/src/Access/QuotaCache.cpp index 43ab4268b0c..af47819b8ce 100644 --- a/src/Access/QuotaCache.cpp +++ b/src/Access/QuotaCache.cpp @@ -301,4 +301,5 @@ std::vector QuotaCache::getAllQuotasUsage() const } return all_usage; } + } diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index c7aec75265c..747c354b28f 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -24,8 +24,8 @@ namespace DB { namespace ErrorCodes { -extern const int BAD_ARGUMENTS; -extern const int NO_ZOOKEEPER; + extern const int BAD_ARGUMENTS; + extern const int NO_ZOOKEEPER; } static UUID parseUUID(const String & text) @@ -46,6 +46,7 @@ ReplicatedAccessStorage::ReplicatedAccessStorage( , zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) , watched_queue(std::make_shared>(std::numeric_limits::max())) + , memory_storage(storage_name_, changes_notifier_, false) , changes_notifier(changes_notifier_) , backup_allowed(allow_backup_) { @@ -59,7 +60,7 @@ ReplicatedAccessStorage::ReplicatedAccessStorage( if (zookeeper_path.front() != '/') zookeeper_path = "/" + zookeeper_path; - initializeZookeeper(); + initZooKeeperWithRetries(/* max_retries= */ 2); } ReplicatedAccessStorage::~ReplicatedAccessStorage() @@ -121,15 +122,14 @@ bool ReplicatedAccessStorage::insertWithID(const UUID & id, const AccessEntityPt const String & name = new_entity->getName(); LOG_DEBUG(getLogger(), "Inserting entity of type {} named {} with id {}", type_info.name, name, toString(id)); - auto zookeeper = get_zookeeper(); + auto zookeeper = getZooKeeper(); bool ok = false; retryOnZooKeeperUserError(10, [&]{ ok = insertZooKeeper(zookeeper, id, new_entity, replace_if_exists, throw_if_exists); }); if (!ok) return false; - std::lock_guard lock{mutex}; - refreshEntityNoLock(zookeeper, id); + refreshEntity(zookeeper, id); return true; } @@ -163,51 +163,79 @@ bool ReplicatedAccessStorage::insertZooKeeper( if (res == Coordination::Error::ZNODEEXISTS) { + if (!throw_if_exists && !replace_if_exists) + return false; /// Couldn't insert a new entity. + + if (throw_if_exists) + { + if (responses[0]->error == Coordination::Error::ZNODEEXISTS) + { + /// To fail with a nice error message, we need info about what already exists. + /// This itself could fail if the conflicting uuid disappears in the meantime. + /// If that happens, then we'll just retry from the start. + String existing_entity_definition = zookeeper->get(entity_path); + + AccessEntityPtr existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path); + AccessEntityType existing_type = existing_entity->getType(); + String existing_name = existing_entity->getName(); + throwIDCollisionCannotInsert(id, type, name, existing_type, existing_name); + } + else + { + /// Couldn't insert the new entity because there is an existing entity with such name. + throwNameCollisionCannotInsert(type, name); + } + } + + assert(replace_if_exists); + Coordination::Requests replace_ops; if (responses[0]->error == Coordination::Error::ZNODEEXISTS) { - /// The UUID already exists, simply fail. - - /// To fail with a nice error message, we need info about what already exists. - /// This itself could fail if the conflicting uuid disappears in the meantime. + /// The UUID is already associated with some existing entity, we will get rid of the conflicting entity first. + /// This itself could fail if the conflicting entity disappears in the meantime. /// If that happens, then we'll just retry from the start. - String existing_entity_definition = zookeeper->get(entity_path); + Coordination::Stat stat; + String existing_entity_definition = zookeeper->get(entity_path, &stat); + auto existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path); + const String & existing_entity_name = existing_entity->getName(); + const AccessEntityType existing_entity_type = existing_entity->getType(); + const AccessEntityTypeInfo existing_entity_type_info = AccessEntityTypeInfo::get(existing_entity_type); + const String existing_name_path = zookeeper_path + "/" + existing_entity_type_info.unique_char + "/" + escapeForFileName(existing_entity_name); - AccessEntityPtr existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path); - AccessEntityType existing_type = existing_entity->getType(); - String existing_name = existing_entity->getName(); - throwIDCollisionCannotInsert(id, type, name, existing_type, existing_name); - } - else if (replace_if_exists) - { - /// The name already exists for this type. - /// If asked to, we need to replace the existing entity. + if (existing_name_path != name_path) + replace_ops.emplace_back(zkutil::makeRemoveRequest(existing_name_path, -1)); - /// First get the uuid of the existing entity - /// This itself could fail if the conflicting name disappears in the meantime. - /// If that happens, then we'll just retry from the start. - Coordination::Stat name_stat; - String existing_entity_uuid = zookeeper->get(name_path, &name_stat); - - const String existing_entity_path = zookeeper_path + "/uuid/" + existing_entity_uuid; - Coordination::Requests replace_ops; - replace_ops.emplace_back(zkutil::makeRemoveRequest(existing_entity_path, -1)); - replace_ops.emplace_back(zkutil::makeCreateRequest(entity_path, new_entity_definition, zkutil::CreateMode::Persistent)); - replace_ops.emplace_back(zkutil::makeSetRequest(name_path, entity_uuid, name_stat.version)); - - /// If this fails, then we'll just retry from the start. - zookeeper->multi(replace_ops); - - /// Everything's fine, the new entity has been inserted instead of an existing entity. - return true; + replace_ops.emplace_back(zkutil::makeSetRequest(entity_path, new_entity_definition, stat.version)); } else { - /// Couldn't insert the new entity because there is an existing entity with such name. - if (throw_if_exists) - throwNameCollisionCannotInsert(type, name); - else - return false; + replace_ops.emplace_back(zkutil::makeCreateRequest(entity_path, new_entity_definition, zkutil::CreateMode::Persistent)); } + + if (responses[1]->error == Coordination::Error::ZNODEEXISTS) + { + /// The name is already associated with some existing entity, we will get rid of the conflicting entity first. + /// This itself could fail if the conflicting entity disappears in the meantime. + /// If that happens, then we'll just retry from the start. + Coordination::Stat stat; + String existing_entity_uuid = zookeeper->get(name_path, &stat); + const String existing_entity_path = zookeeper_path + "/uuid/" + existing_entity_uuid; + + if (existing_entity_path != entity_path) + replace_ops.emplace_back(zkutil::makeRemoveRequest(existing_entity_path, -1)); + + replace_ops.emplace_back(zkutil::makeSetRequest(name_path, entity_uuid, stat.version)); + } + else + { + replace_ops.emplace_back(zkutil::makeCreateRequest(name_path, entity_uuid, zkutil::CreateMode::Persistent)); + } + + /// If this fails, then we'll just retry from the start. + zookeeper->multi(replace_ops); + + /// Everything's fine, the new entity has been inserted instead of an existing entity. + return true; } /// If this fails, then we'll just retry from the start. @@ -221,7 +249,7 @@ bool ReplicatedAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exis { LOG_DEBUG(getLogger(), "Removing entity {}", toString(id)); - auto zookeeper = get_zookeeper(); + auto zookeeper = getZooKeeper(); bool ok = false; retryOnZooKeeperUserError(10, [&] { ok = removeZooKeeper(zookeeper, id, throw_if_not_exists); }); @@ -273,15 +301,14 @@ bool ReplicatedAccessStorage::updateImpl(const UUID & id, const UpdateFunc & upd { LOG_DEBUG(getLogger(), "Updating entity {}", toString(id)); - auto zookeeper = get_zookeeper(); + auto zookeeper = getZooKeeper(); bool ok = false; retryOnZooKeeperUserError(10, [&] { ok = updateZooKeeper(zookeeper, id, update_func, throw_if_not_exists); }); if (!ok) return false; - std::lock_guard lock{mutex}; - refreshEntityNoLock(zookeeper, id); + refreshEntity(zookeeper, id); return true; } @@ -349,50 +376,110 @@ void ReplicatedAccessStorage::runWatchingThread() { LOG_DEBUG(getLogger(), "Started watching thread"); setThreadName("ReplACLWatch"); + while (watching) { + bool refreshed = false; try { - if (!initialized) - initializeZookeeper(); - if (refresh()) - changes_notifier.sendNotifications(); + initZooKeeperIfNeeded(); + refreshed = refresh(); } catch (...) { - tryLogCurrentException(getLogger(), "Unexpected error, will try to restart worker thread:"); + tryLogCurrentException(getLogger(), "Will try to restart watching thread after error"); resetAfterError(); sleepForSeconds(5); + continue; + } + + if (refreshed) + { + try + { + changes_notifier.sendNotifications(); + } + catch (...) + { + tryLogCurrentException(getLogger(), "Error while sending notifications"); + } } } } void ReplicatedAccessStorage::resetAfterError() { - initialized = false; - - UUID id; - while (watched_queue->tryPop(id)) {} - - std::lock_guard lock{mutex}; - for (const auto type : collections::range(AccessEntityType::MAX)) - entries_by_name_and_type[static_cast(type)].clear(); - entries_by_id.clear(); + /// Make watching thread reinitialize ZooKeeper and reread everything. + std::lock_guard lock{cached_zookeeper_mutex}; + cached_zookeeper = nullptr; } -void ReplicatedAccessStorage::initializeZookeeper() +void ReplicatedAccessStorage::initZooKeeperWithRetries(size_t max_retries) { - assert(!initialized); - auto zookeeper = get_zookeeper(); + for (size_t attempt = 0; attempt < max_retries; ++attempt) + { + try + { + initZooKeeperIfNeeded(); + break; /// If we're here the initialization has been successful. + } + catch (const Exception & e) + { + bool need_another_attempt = false; - if (!zookeeper) - throw Exception("Can't have Replicated access without ZooKeeper", ErrorCodes::NO_ZOOKEEPER); + if (const auto * coordination_exception = dynamic_cast(&e); + coordination_exception && Coordination::isHardwareError(coordination_exception->code)) + { + /// In case of a network error we'll try to initialize again. + LOG_ERROR(getLogger(), "Initialization failed. Error: {}", e.message()); + need_another_attempt = (attempt + 1 < max_retries); + } - createRootNodes(zookeeper); + if (!need_another_attempt) + throw; + } + } +} - refreshEntities(zookeeper); +void ReplicatedAccessStorage::initZooKeeperIfNeeded() +{ + getZooKeeper(); +} - initialized = true; +zkutil::ZooKeeperPtr ReplicatedAccessStorage::getZooKeeper() +{ + std::lock_guard lock{cached_zookeeper_mutex}; + return getZooKeeperNoLock(); +} + +zkutil::ZooKeeperPtr ReplicatedAccessStorage::getZooKeeperNoLock() +{ + if (!cached_zookeeper || cached_zookeeper->expired()) + { + auto zookeeper = get_zookeeper(); + if (!zookeeper) + throw Exception("Can't have Replicated access without ZooKeeper", ErrorCodes::NO_ZOOKEEPER); + + /// It's possible that we connected to different [Zoo]Keeper instance + /// so we may read a bit stale state. + zookeeper->sync(zookeeper_path); + + createRootNodes(zookeeper); + refreshEntities(zookeeper, /* all= */ true); + cached_zookeeper = zookeeper; + } + return cached_zookeeper; +} + +void ReplicatedAccessStorage::reload(ReloadMode reload_mode) +{ + if (reload_mode != ReloadMode::ALL) + return; + + /// Reinitialize ZooKeeper and reread everything. + std::lock_guard lock{cached_zookeeper_mutex}; + cached_zookeeper = nullptr; + getZooKeeperNoLock(); } void ReplicatedAccessStorage::createRootNodes(const zkutil::ZooKeeperPtr & zookeeper) @@ -414,10 +501,10 @@ bool ReplicatedAccessStorage::refresh() if (!watched_queue->tryPop(id, /* timeout_ms: */ 10000)) return false; - auto zookeeper = get_zookeeper(); + auto zookeeper = getZooKeeper(); if (id == UUIDHelpers::Nil) - refreshEntities(zookeeper); + refreshEntities(zookeeper, /* all= */ false); else refreshEntity(zookeeper, id); @@ -425,10 +512,16 @@ bool ReplicatedAccessStorage::refresh() } -void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zookeeper) +void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zookeeper, bool all) { LOG_DEBUG(getLogger(), "Refreshing entities list"); + if (all) + { + /// It doesn't make sense to keep the queue because we will reread everything in this function. + watched_queue->clear(); + } + const String zookeeper_uuids_path = zookeeper_path + "/uuid"; auto watch_entities_list = [watched_queue = watched_queue](const Coordination::WatchResponse &) { @@ -437,185 +530,129 @@ void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zooke Coordination::Stat stat; const auto entity_uuid_strs = zookeeper->getChildrenWatch(zookeeper_uuids_path, &stat, watch_entities_list); - std::unordered_set entity_uuids; + std::vector entity_uuids; entity_uuids.reserve(entity_uuid_strs.size()); for (const String & entity_uuid_str : entity_uuid_strs) - entity_uuids.insert(parseUUID(entity_uuid_str)); + entity_uuids.emplace_back(parseUUID(entity_uuid_str)); std::lock_guard lock{mutex}; - std::vector entities_to_remove; - /// Locally remove entities that were removed from ZooKeeper - for (const auto & pair : entries_by_id) + if (all) { - const UUID & entity_uuid = pair.first; - if (!entity_uuids.contains(entity_uuid)) - entities_to_remove.push_back(entity_uuid); + /// all=true means we read & parse all access entities from ZooKeeper. + std::vector> entities; + for (const auto & uuid : entity_uuids) + { + if (auto entity = tryReadEntityFromZooKeeper(zookeeper, uuid)) + entities.emplace_back(uuid, entity); + } + memory_storage.setAll(entities); } - for (const auto & entity_uuid : entities_to_remove) - removeEntityNoLock(entity_uuid); - - /// Locally add entities that were added to ZooKeeper - for (const auto & entity_uuid : entity_uuids) + else { - const auto it = entries_by_id.find(entity_uuid); - if (it == entries_by_id.end()) - refreshEntityNoLock(zookeeper, entity_uuid); + /// all=false means we read & parse only new access entities from ZooKeeper. + memory_storage.removeAllExcept(entity_uuids); + for (const auto & uuid : entity_uuids) + { + if (!memory_storage.exists(uuid)) + refreshEntityNoLock(zookeeper, uuid); + } } LOG_DEBUG(getLogger(), "Refreshing entities list finished"); } + void ReplicatedAccessStorage::refreshEntity(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) { + LOG_DEBUG(getLogger(), "Refreshing entity {}", toString(id)); + + auto entity = tryReadEntityFromZooKeeper(zookeeper, id); + std::lock_guard lock{mutex}; - refreshEntityNoLock(zookeeper, id); + + if (entity) + setEntityNoLock(id, entity); + else + removeEntityNoLock(id); } void ReplicatedAccessStorage::refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) { LOG_DEBUG(getLogger(), "Refreshing entity {}", toString(id)); + auto entity = tryReadEntityFromZooKeeper(zookeeper, id); + if (entity) + setEntityNoLock(id, entity); + else + removeEntityNoLock(id); +} + +AccessEntityPtr ReplicatedAccessStorage::tryReadEntityFromZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) const +{ const auto watch_entity = [watched_queue = watched_queue, id](const Coordination::WatchResponse & response) { if (response.type == Coordination::Event::CHANGED) [[maybe_unused]] bool push_result = watched_queue->push(id); }; + Coordination::Stat entity_stat; const String entity_path = zookeeper_path + "/uuid/" + toString(id); String entity_definition; - const bool exists = zookeeper->tryGetWatch(entity_path, entity_definition, &entity_stat, watch_entity); - if (exists) + bool exists = zookeeper->tryGetWatch(entity_path, entity_definition, &entity_stat, watch_entity); + if (!exists) + return nullptr; + + try { - const AccessEntityPtr entity = deserializeAccessEntity(entity_definition, entity_path); - setEntityNoLock(id, entity); + return deserializeAccessEntity(entity_definition, entity_path); } - else + catch (...) { - removeEntityNoLock(id); + tryLogCurrentException(getLogger(), "Error while reading the definition of " + toString(id)); + return nullptr; } } - void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntityPtr & entity) { LOG_DEBUG(getLogger(), "Setting id {} to entity named {}", toString(id), entity->getName()); - const AccessEntityType type = entity->getType(); - const String & name = entity->getName(); - - /// If the type+name already exists and is a different entity, remove old entity - auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; - if (auto it = entries_by_name.find(name); it != entries_by_name.end() && it->second->id != id) - { - removeEntityNoLock(it->second->id); - } - - /// If the entity already exists under a different type+name, remove old type+name - bool existed_before = false; - if (auto it = entries_by_id.find(id); it != entries_by_id.end()) - { - existed_before = true; - const AccessEntityPtr & existing_entity = it->second.entity; - const AccessEntityType existing_type = existing_entity->getType(); - const String & existing_name = existing_entity->getName(); - if (existing_type != type || existing_name != name) - { - auto & existing_entries_by_name = entries_by_name_and_type[static_cast(existing_type)]; - existing_entries_by_name.erase(existing_name); - } - } - - auto & entry = entries_by_id[id]; - entry.id = id; - entry.entity = entity; - entries_by_name[name] = &entry; - - if (initialized) - { - if (existed_before) - changes_notifier.onEntityUpdated(id, entity); - else - changes_notifier.onEntityAdded(id, entity); - } + memory_storage.insertWithID(id, entity, /* replace_if_exists= */ true, /* throw_if_exists= */ false); } void ReplicatedAccessStorage::removeEntityNoLock(const UUID & id) { LOG_DEBUG(getLogger(), "Removing entity with id {}", toString(id)); - const auto it = entries_by_id.find(id); - if (it == entries_by_id.end()) - { - LOG_DEBUG(getLogger(), "Id {} not found, ignoring removal", toString(id)); - return; - } - - const Entry & entry = it->second; - const AccessEntityType type = entry.entity->getType(); - const String & name = entry.entity->getName(); - - auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; - const auto name_it = entries_by_name.find(name); - if (name_it == entries_by_name.end()) - LOG_WARNING(getLogger(), "Entity {} not found in names, ignoring removal of name", toString(id)); - else if (name_it->second != &(it->second)) - LOG_WARNING(getLogger(), "Name {} not pointing to entity {}, ignoring removal of name", name, toString(id)); - else - entries_by_name.erase(name); - - UUID removed_id = id; - entries_by_id.erase(id); - LOG_DEBUG(getLogger(), "Removed entity with id {}", toString(id)); - - changes_notifier.onEntityRemoved(removed_id, type); + memory_storage.remove(id, /* throw_if_not_exists= */ false); } std::optional ReplicatedAccessStorage::findImpl(AccessEntityType type, const String & name) const { std::lock_guard lock{mutex}; - const auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; - const auto it = entries_by_name.find(name); - if (it == entries_by_name.end()) - return {}; - - const Entry * entry = it->second; - return entry->id; + return memory_storage.find(type, name); } std::vector ReplicatedAccessStorage::findAllImpl(AccessEntityType type) const { std::lock_guard lock{mutex}; - std::vector result; - result.reserve(entries_by_id.size()); - for (const auto & [id, entry] : entries_by_id) - if (entry.entity->isTypeOf(type)) - result.emplace_back(id); - return result; + return memory_storage.findAll(type); } bool ReplicatedAccessStorage::exists(const UUID & id) const { std::lock_guard lock{mutex}; - return entries_by_id.contains(id); + return memory_storage.exists(id); } AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const { std::lock_guard lock{mutex}; - const auto it = entries_by_id.find(id); - if (it == entries_by_id.end()) - { - if (throw_if_not_exists) - throwNotFound(id); - else - return nullptr; - } - const Entry & entry = it->second; - return entry.entity; + return memory_storage.read(id, throw_if_not_exists); } diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h index 6311e2ac7c0..d9d4b628f8d 100644 --- a/src/Access/ReplicatedAccessStorage.h +++ b/src/Access/ReplicatedAccessStorage.h @@ -1,20 +1,13 @@ #pragma once #include -#include -#include -#include -#include - -#include -#include #include #include #include #include -#include +#include namespace DB @@ -34,6 +27,7 @@ public: void startPeriodicReloading() override { startWatchingThread(); } void stopPeriodicReloading() override { stopWatchingThread(); } + void reload(ReloadMode reload_mode) override; bool exists(const UUID & id) const override; @@ -43,9 +37,10 @@ public: private: String zookeeper_path; - zkutil::GetZooKeeper get_zookeeper; + const zkutil::GetZooKeeper get_zookeeper; - std::atomic initialized = false; + zkutil::ZooKeeperPtr cached_zookeeper TSA_GUARDED_BY(cached_zookeeper_mutex); + std::mutex cached_zookeeper_mutex; std::atomic watching = false; ThreadFromGlobalPool watching_thread; @@ -60,7 +55,10 @@ private: bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists); bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); - void initializeZookeeper(); + void initZooKeeperWithRetries(size_t max_retries); + void initZooKeeperIfNeeded(); + zkutil::ZooKeeperPtr getZooKeeper(); + zkutil::ZooKeeperPtr getZooKeeperNoLock() TSA_REQUIRES(cached_zookeeper_mutex); void createRootNodes(const zkutil::ZooKeeperPtr & zookeeper); void startWatchingThread(); @@ -70,27 +68,21 @@ private: void resetAfterError(); bool refresh(); - void refreshEntities(const zkutil::ZooKeeperPtr & zookeeper); + void refreshEntities(const zkutil::ZooKeeperPtr & zookeeper, bool all); void refreshEntity(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id); void refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) TSA_REQUIRES(mutex); + AccessEntityPtr tryReadEntityFromZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) const; void setEntityNoLock(const UUID & id, const AccessEntityPtr & entity) TSA_REQUIRES(mutex); void removeEntityNoLock(const UUID & id) TSA_REQUIRES(mutex); - struct Entry - { - UUID id; - AccessEntityPtr entity; - }; - std::optional findImpl(AccessEntityType type, const String & name) const override; std::vector findAllImpl(AccessEntityType type) const override; AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; mutable std::mutex mutex; - std::unordered_map entries_by_id TSA_GUARDED_BY(mutex); - std::unordered_map entries_by_name_and_type[static_cast(AccessEntityType::MAX)] TSA_GUARDED_BY(mutex); + MemoryAccessStorage memory_storage TSA_GUARDED_BY(mutex); AccessChangesNotifier & changes_notifier; - bool backup_allowed = false; + const bool backup_allowed = false; }; } diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 34f2e10dc83..d97a78c78ab 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -35,88 +35,43 @@ void SettingsConstraints::clear() constraints.clear(); } - -void SettingsConstraints::setMinValue(std::string_view setting_name, const Field & min_value) +void SettingsConstraints::set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability) { - getConstraintRef(setting_name).min_value = Settings::castValueUtil(setting_name, min_value); + auto & constraint = constraints[setting_name]; + if (!min_value.isNull()) + constraint.min_value = Settings::castValueUtil(setting_name, min_value); + if (!max_value.isNull()) + constraint.max_value = Settings::castValueUtil(setting_name, max_value); + constraint.writability = writability; } -Field SettingsConstraints::getMinValue(std::string_view setting_name) const +void SettingsConstraints::get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const { - const auto * ptr = tryGetConstraint(setting_name); - if (ptr) - return ptr->min_value; - else - return {}; -} - - -void SettingsConstraints::setMaxValue(std::string_view setting_name, const Field & max_value) -{ - getConstraintRef(setting_name).max_value = Settings::castValueUtil(setting_name, max_value); -} - -Field SettingsConstraints::getMaxValue(std::string_view setting_name) const -{ - const auto * ptr = tryGetConstraint(setting_name); - if (ptr) - return ptr->max_value; - else - return {}; -} - - -void SettingsConstraints::setReadOnly(std::string_view setting_name, bool read_only) -{ - getConstraintRef(setting_name).read_only = read_only; -} - -bool SettingsConstraints::isReadOnly(std::string_view setting_name) const -{ - const auto * ptr = tryGetConstraint(setting_name); - if (ptr) - return ptr->read_only; - else - return false; -} - - -void SettingsConstraints::set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only) -{ - auto & ref = getConstraintRef(setting_name); - ref.min_value = Settings::castValueUtil(setting_name, min_value); - ref.max_value = Settings::castValueUtil(setting_name, max_value); - ref.read_only = read_only; -} - -void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const -{ - const auto * ptr = tryGetConstraint(setting_name); - if (ptr) - { - min_value = ptr->min_value; - max_value = ptr->max_value; - read_only = ptr->read_only; - } - else - { - min_value = Field{}; - max_value = Field{}; - read_only = false; - } + auto checker = getChecker(current_settings, setting_name); + min_value = checker.constraint.min_value; + max_value = checker.constraint.max_value; + writability = checker.constraint.writability; } void SettingsConstraints::merge(const SettingsConstraints & other) { - for (const auto & [other_name, other_constraint] : other.constraints) + if (access_control->doesSettingsConstraintsReplacePrevious()) { - auto & constraint = getConstraintRef(other_name); - if (!other_constraint.min_value.isNull()) - constraint.min_value = other_constraint.min_value; - if (!other_constraint.max_value.isNull()) - constraint.max_value = other_constraint.max_value; - if (other_constraint.read_only) - constraint.read_only = true; + for (const auto & [other_name, other_constraint] : other.constraints) + constraints[other_name] = other_constraint; + } + else + { + for (const auto & [other_name, other_constraint] : other.constraints) + { + auto & constraint = constraints[other_name]; + if (!other_constraint.min_value.isNull()) + constraint.min_value = other_constraint.min_value; + if (!other_constraint.max_value.isNull()) + constraint.max_value = other_constraint.max_value; + if (other_constraint.writability == SettingConstraintWritability::CONST) + constraint.writability = SettingConstraintWritability::CONST; // NOTE: In this mode flag cannot be overridden to be false + } } } @@ -180,26 +135,6 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh } }; - bool cannot_compare = false; - auto less = [&](const Field & left, const Field & right) - { - cannot_compare = false; - if (reaction == THROW_ON_VIOLATION) - return applyVisitor(FieldVisitorAccurateLess{}, left, right); - else - { - try - { - return applyVisitor(FieldVisitorAccurateLess{}, left, right); - } - catch (...) - { - cannot_compare = true; - return false; - } - } - }; - if (reaction == THROW_ON_VIOLATION) { try @@ -239,115 +174,119 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh return false; } - if (!current_settings.allow_ddl && setting_name == "allow_ddl") + return getChecker(current_settings, setting_name).check(change, new_value, reaction); +} + +bool SettingsConstraints::Checker::check(SettingChange & change, const Field & new_value, ReactionOnViolation reaction) const +{ + const String & setting_name = change.name; + + auto less_or_cannot_compare = [=](const Field & left, const Field & right) { if (reaction == THROW_ON_VIOLATION) - throw Exception("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED); + return applyVisitor(FieldVisitorAccurateLess{}, left, right); else - return false; - } - - /** The `readonly` value is understood as follows: - * 0 - everything allowed. - * 1 - only read queries can be made; you can not change the settings. - * 2 - You can only do read queries and you can change the settings, except for the `readonly` setting. - */ - if (current_settings.readonly == 1) - { - if (reaction == THROW_ON_VIOLATION) - throw Exception("Cannot modify '" + setting_name + "' setting in readonly mode", ErrorCodes::READONLY); - else - return false; - } - - if (current_settings.readonly > 1 && setting_name == "readonly") - { - if (reaction == THROW_ON_VIOLATION) - throw Exception("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY); - else - return false; - } - - const Constraint * constraint = tryGetConstraint(setting_name); - if (constraint) - { - if (constraint->read_only) { - if (reaction == THROW_ON_VIOLATION) - throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); - else - return false; - } - - const Field & min_value = constraint->min_value; - const Field & max_value = constraint->max_value; - if (!min_value.isNull() && !max_value.isNull() && (less(max_value, min_value) || cannot_compare)) - { - if (reaction == THROW_ON_VIOLATION) - throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); - else - return false; - } - - if (!min_value.isNull() && (less(new_value, min_value) || cannot_compare)) - { - if (reaction == THROW_ON_VIOLATION) + try { - throw Exception( - "Setting " + setting_name + " shouldn't be less than " + applyVisitor(FieldVisitorToString(), constraint->min_value), - ErrorCodes::SETTING_CONSTRAINT_VIOLATION); + return applyVisitor(FieldVisitorAccurateLess{}, left, right); } - else - change.value = min_value; - } - - if (!max_value.isNull() && (less(max_value, new_value) || cannot_compare)) - { - if (reaction == THROW_ON_VIOLATION) + catch (...) { - throw Exception( - "Setting " + setting_name + " shouldn't be greater than " + applyVisitor(FieldVisitorToString(), constraint->max_value), - ErrorCodes::SETTING_CONSTRAINT_VIOLATION); + return true; } - else - change.value = max_value; } + }; + + if (!explain.empty()) + { + if (reaction == THROW_ON_VIOLATION) + throw Exception(explain, code); + else + return false; + } + + if (constraint.writability == SettingConstraintWritability::CONST) + { + if (reaction == THROW_ON_VIOLATION) + throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); + else + return false; + } + + const auto & min_value = constraint.min_value; + const auto & max_value = constraint.max_value; + + if (!min_value.isNull() && !max_value.isNull() && less_or_cannot_compare(max_value, min_value)) + { + if (reaction == THROW_ON_VIOLATION) + throw Exception("Setting " + setting_name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION); + else + return false; + } + + if (!min_value.isNull() && less_or_cannot_compare(new_value, min_value)) + { + if (reaction == THROW_ON_VIOLATION) + { + throw Exception( + "Setting " + setting_name + " shouldn't be less than " + applyVisitor(FieldVisitorToString(), min_value), + ErrorCodes::SETTING_CONSTRAINT_VIOLATION); + } + else + change.value = min_value; + } + + if (!max_value.isNull() && less_or_cannot_compare(max_value, new_value)) + { + if (reaction == THROW_ON_VIOLATION) + { + throw Exception( + "Setting " + setting_name + " shouldn't be greater than " + applyVisitor(FieldVisitorToString(), max_value), + ErrorCodes::SETTING_CONSTRAINT_VIOLATION); + } + else + change.value = max_value; } return true; } - -SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(std::string_view setting_name) +SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & current_settings, std::string_view setting_name) const { + if (!current_settings.allow_ddl && setting_name == "allow_ddl") + return Checker("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED); + + /** The `readonly` value is understood as follows: + * 0 - no read-only restrictions. + * 1 - only read requests, as well as changing settings with `changable_in_readonly` flag. + * 2 - only read requests, as well as changing settings, except for the `readonly` setting. + */ + + if (current_settings.readonly > 1 && setting_name == "readonly") + return Checker("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY); + auto it = constraints.find(setting_name); - if (it == constraints.end()) + if (current_settings.readonly == 1) { - auto setting_name_ptr = std::make_shared(setting_name); - Constraint new_constraint; - new_constraint.setting_name = setting_name_ptr; - it = constraints.emplace(*setting_name_ptr, std::move(new_constraint)).first; + if (it == constraints.end() || it->second.writability != SettingConstraintWritability::CHANGEABLE_IN_READONLY) + return Checker("Cannot modify '" + String(setting_name) + "' setting in readonly mode", ErrorCodes::READONLY); } - return it->second; + else // For both readonly=0 and readonly=2 + { + if (it == constraints.end()) + return Checker(); // Allowed + } + return Checker(it->second); } -const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(std::string_view setting_name) const -{ - auto it = constraints.find(setting_name); - if (it == constraints.end()) - return nullptr; - return &it->second; -} - - bool SettingsConstraints::Constraint::operator==(const Constraint & other) const { - return (read_only == other.read_only) && (min_value == other.min_value) && (max_value == other.max_value) - && (*setting_name == *other.setting_name); + return writability == other.writability && min_value == other.min_value && max_value == other.max_value; } bool operator ==(const SettingsConstraints & left, const SettingsConstraints & right) { - return (left.constraints == right.constraints); + return left.constraints == right.constraints; } } diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 645a690e051..822bf42861b 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -1,9 +1,9 @@ #pragma once +#include #include #include - namespace Poco::Util { class AbstractConfiguration; @@ -35,18 +35,22 @@ class AccessControl; * 20000000000 * * - * + * * + * + * + * * * * * * This class also checks that we are not in the read-only mode. * If a setting cannot be change due to the read-only mode this class throws an exception. - * The value of `readonly` value is understood as follows: - * 0 - everything allowed. - * 1 - only read queries can be made; you can not change the settings. - * 2 - you can only do read queries and you can change the settings, except for the `readonly` setting. + * The value of `readonly` is understood as follows: + * 0 - not read-only mode, no additional checks. + * 1 - only read queries, as well as changing settings with flag. + * 2 - only read queries and you can change the settings, except for the `readonly` setting. + * */ class SettingsConstraints { @@ -61,17 +65,8 @@ public: void clear(); bool empty() const { return constraints.empty(); } - void setMinValue(std::string_view setting_name, const Field & min_value); - Field getMinValue(std::string_view setting_name) const; - - void setMaxValue(std::string_view setting_name, const Field & max_value); - Field getMaxValue(std::string_view setting_name) const; - - void setReadOnly(std::string_view setting_name, bool read_only); - bool isReadOnly(std::string_view setting_name) const; - - void set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only); - void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const; + void set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability); + void get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const; void merge(const SettingsConstraints & other); @@ -87,10 +82,15 @@ public: friend bool operator !=(const SettingsConstraints & left, const SettingsConstraints & right) { return !(left == right); } private: + enum ReactionOnViolation + { + THROW_ON_VIOLATION, + CLAMP_ON_VIOLATION, + }; + struct Constraint { - std::shared_ptr setting_name; - bool read_only = false; + SettingConstraintWritability writability = SettingConstraintWritability::WRITABLE; Field min_value; Field max_value; @@ -98,18 +98,53 @@ private: bool operator !=(const Constraint & other) const { return !(*this == other); } }; - enum ReactionOnViolation + struct Checker { - THROW_ON_VIOLATION, - CLAMP_ON_VIOLATION, + Constraint constraint; + String explain; + int code = 0; + + // Allows everything + Checker() = default; + + // Forbidden with explanation + Checker(const String & explain_, int code_) + : constraint{.writability = SettingConstraintWritability::CONST} + , explain(explain_) + , code(code_) + {} + + // Allow or forbid depending on range defined by constraint, also used to return stored constraint + explicit Checker(const Constraint & constraint_) + : constraint(constraint_) + {} + + // Perform checking + bool check(SettingChange & change, const Field & new_value, ReactionOnViolation reaction) const; }; + + struct StringHash + { + using is_transparent = void; + size_t operator()(std::string_view txt) const + { + return std::hash{}(txt); + } + size_t operator()(const String & txt) const + { + return std::hash{}(txt); + } + }; + bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const; - Constraint & getConstraintRef(std::string_view setting_name); - const Constraint * tryGetConstraint(std::string_view setting_name) const; + Checker getChecker(const Settings & current_settings, std::string_view setting_name) const; - std::unordered_map constraints; - const AccessControl * access_control = nullptr; + // Special container for heterogeneous lookups: to avoid `String` construction during `find(std::string_view)` + using Constraints = std::unordered_map>; + Constraints constraints; + + const AccessControl * access_control; }; } diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 465f26f37d9..474ffec0d21 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,10 @@ namespace constexpr const char ALLOW_BACKUP_SETTING_NAME[] = "allow_backup"; } +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} SettingsProfileElement::SettingsProfileElement(const ASTSettingsProfileElement & ast) { @@ -46,17 +51,20 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A { setting_name = ast.setting_name; - /// Optionally check if a setting with that name is allowed. if (access_control) { + /// Check if a setting with that name is allowed. if (setting_name != ALLOW_BACKUP_SETTING_NAME) access_control->checkSettingNameIsAllowed(setting_name); + /// Check if a CHANGEABLE_IN_READONLY is allowed. + if (ast.writability == SettingConstraintWritability::CHANGEABLE_IN_READONLY && !access_control->doesSettingsConstraintsReplacePrevious()) + throw Exception("CHANGEABLE_IN_READONLY for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); } value = ast.value; min_value = ast.min_value; max_value = ast.max_value; - readonly = ast.readonly; + writability = ast.writability; if (!value.isNull()) value = Settings::castValueUtil(setting_name, value); @@ -80,7 +88,7 @@ std::shared_ptr SettingsProfileElement::toAST() const ast->value = value; ast->min_value = min_value; ast->max_value = max_value; - ast->readonly = readonly; + ast->writability = writability; return ast; } @@ -101,7 +109,7 @@ std::shared_ptr SettingsProfileElement::toASTWithName ast->value = value; ast->min_value = min_value; ast->max_value = max_value; - ast->readonly = readonly; + ast->writability = writability; return ast; } @@ -205,17 +213,12 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC { SettingsConstraints res{access_control}; for (const auto & elem : *this) - { - if (!elem.setting_name.empty() && (elem.setting_name != ALLOW_BACKUP_SETTING_NAME)) - { - if (!elem.min_value.isNull()) - res.setMinValue(elem.setting_name, elem.min_value); - if (!elem.max_value.isNull()) - res.setMaxValue(elem.setting_name, elem.max_value); - if (elem.readonly) - res.setReadOnly(elem.setting_name, *elem.readonly); - } - } + if (!elem.setting_name.empty() && elem.setting_name != ALLOW_BACKUP_SETTING_NAME) + res.set( + elem.setting_name, + elem.min_value, + elem.max_value, + elem.writability ? *elem.writability : SettingConstraintWritability::WRITABLE); return res; } diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index a4124826b40..c02e9947d61 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -25,9 +26,9 @@ struct SettingsProfileElement Field value; Field min_value; Field max_value; - std::optional readonly; + std::optional writability; - auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, readonly); } + auto toTuple() const { return std::tie(parent_profile, setting_name, value, min_value, max_value, writability); } friend bool operator==(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() == rhs.toTuple(); } friend bool operator!=(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return !(lhs == rhs); } friend bool operator <(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() < rhs.toTuple(); } diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 1d755fdf1da..28ef4ebaa7a 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -441,17 +441,32 @@ namespace String path_to_name = path_to_constraints + "." + setting_name; config.keys(path_to_name, constraint_types); + size_t writability_count = 0; for (const String & constraint_type : constraint_types) { if (constraint_type == "min") profile_element.min_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); else if (constraint_type == "max") profile_element.max_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); - else if (constraint_type == "readonly") - profile_element.readonly = true; + else if (constraint_type == "readonly" || constraint_type == "const") + { + writability_count++; + profile_element.writability = SettingConstraintWritability::CONST; + } + else if (constraint_type == "changeable_in_readonly") + { + writability_count++; + if (access_control.doesSettingsConstraintsReplacePrevious()) + profile_element.writability = SettingConstraintWritability::CHANGEABLE_IN_READONLY; + else + throw Exception("Setting changeable_in_readonly for " + setting_name + " is not allowed unless settings_constraints_replace_previous is enabled", ErrorCodes::NOT_IMPLEMENTED); + } else throw Exception("Setting " + constraint_type + " value for " + setting_name + " isn't supported", ErrorCodes::NOT_IMPLEMENTED); } + if (writability_count > 1) + throw Exception("Not more than one constraint writability specifier (const/readonly/changeable_in_readonly) is allowed for " + setting_name, ErrorCodes::NOT_IMPLEMENTED); + profile_elements.push_back(std::move(profile_element)); } @@ -635,13 +650,6 @@ void UsersConfigAccessStorage::load( /* already_loaded = */ false); } -void UsersConfigAccessStorage::reload() -{ - std::lock_guard lock{load_mutex}; - if (config_reloader) - config_reloader->reload(); -} - void UsersConfigAccessStorage::startPeriodicReloading() { std::lock_guard lock{load_mutex}; @@ -656,6 +664,13 @@ void UsersConfigAccessStorage::stopPeriodicReloading() config_reloader->stop(); } +void UsersConfigAccessStorage::reload(ReloadMode /* reload_mode */) +{ + std::lock_guard lock{load_mutex}; + if (config_reloader) + config_reloader->reload(); +} + std::optional UsersConfigAccessStorage::findImpl(AccessEntityType type, const String & name) const { return memory_storage.find(type, name); diff --git a/src/Access/UsersConfigAccessStorage.h b/src/Access/UsersConfigAccessStorage.h index 3fa8b4185a8..b533ccbf200 100644 --- a/src/Access/UsersConfigAccessStorage.h +++ b/src/Access/UsersConfigAccessStorage.h @@ -38,9 +38,9 @@ public: const String & preprocessed_dir = {}, const zkutil::GetZooKeeper & get_zookeeper_function = {}); - void reload() override; void startPeriodicReloading() override; void stopPeriodicReloading() override; + void reload(ReloadMode reload_mode) override; bool exists(const UUID & id) const override; diff --git a/src/AggregateFunctions/AggregateFunctionArray.h b/src/AggregateFunctions/AggregateFunctionArray.h index 85e0dfc8050..abefe8e0de1 100644 --- a/src/AggregateFunctions/AggregateFunctionArray.h +++ b/src/AggregateFunctions/AggregateFunctionArray.h @@ -64,6 +64,11 @@ public: return nested_func->isVersioned(); } + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + size_t getDefaultVersion() const override { return nested_func->getDefaultVersion(); @@ -79,6 +84,11 @@ public: nested_func->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + nested_func->destroyUpToState(place); + } + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); diff --git a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp b/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp index 35654c08659..89ffdfa6109 100644 --- a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp +++ b/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp @@ -1,12 +1,18 @@ -#include - +#include #include #include #include +#include +#include +#include +#include +#include +#include namespace DB { + struct Settings; namespace ErrorCodes @@ -15,6 +21,136 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } +/** The function takes arguments x1, x2, ... xn, y. All arguments are bool. + * x arguments represents the fact that some category is true. + * + * It calculates how many times y was true and how many times y was false when every n-th category was true + * and the total number of times y was true and false. + * + * So, the size of the state is (n + 1) * 2 cells. + */ +class AggregateFunctionCategoricalIV final : public IAggregateFunctionHelper +{ +private: + using Counter = UInt64; + size_t category_count; + + static Counter & counter(AggregateDataPtr __restrict place, size_t i, bool what) + { + return reinterpret_cast(place)[i * 2 + (what ? 1 : 0)]; + } + + static const Counter & counter(ConstAggregateDataPtr __restrict place, size_t i, bool what) + { + return reinterpret_cast(place)[i * 2 + (what ? 1 : 0)]; + } + +public: + AggregateFunctionCategoricalIV(const DataTypes & arguments_, const Array & params_) : + IAggregateFunctionHelper{arguments_, params_}, + category_count{arguments_.size() - 1} + { + // notice: argument types has been checked before + } + + String getName() const override + { + return "categoricalInformationValue"; + } + + bool allocatesMemoryInArena() const override { return false; } + + void create(AggregateDataPtr __restrict place) const override + { + memset(place, 0, sizeOfData()); + } + + void destroy(AggregateDataPtr __restrict) const noexcept override + { + // nothing + } + + bool hasTrivialDestructor() const override + { + return true; + } + + size_t sizeOfData() const override + { + return sizeof(Counter) * (category_count + 1) * 2; + } + + size_t alignOfData() const override + { + return alignof(Counter); + } + + void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override + { + const auto * y_col = static_cast(columns[category_count]); + bool y = y_col->getData()[row_num]; + + for (size_t i = 0; i < category_count; ++i) + { + const auto * x_col = static_cast(columns[i]); + bool x = x_col->getData()[row_num]; + + if (x) + ++counter(place, i, y); + } + + ++counter(place, category_count, y); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override + { + for (size_t i = 0; i <= category_count; ++i) + { + counter(place, i, false) += counter(rhs, i, false); + counter(place, i, true) += counter(rhs, i, true); + } + } + + void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override + { + buf.write(place, sizeOfData()); + } + + void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override + { + buf.read(place, sizeOfData()); + } + + DataTypePtr getReturnType() const override + { + return std::make_shared( + std::make_shared>()); + } + + void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override /// NOLINT + { + auto & col = static_cast(to); + auto & data_col = static_cast(col.getData()); + auto & offset_col = static_cast(col.getOffsetsColumn()); + + data_col.reserve(data_col.size() + category_count); + + Float64 sum_no = static_cast(counter(place, category_count, false)); + Float64 sum_yes = static_cast(counter(place, category_count, true)); + + for (size_t i = 0; i < category_count; ++i) + { + Float64 no = static_cast(counter(place, i, false)); + Float64 yes = static_cast(counter(place, i, true)); + + data_col.insertValue((no / sum_no - yes / sum_yes) * (log((no / sum_no) / (yes / sum_yes)))); + } + + offset_col.insertValue(data_col.size()); + } +}; + + namespace { @@ -39,16 +175,15 @@ AggregateFunctionPtr createAggregateFunctionCategoricalIV( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - return std::make_shared>(arguments, params); + return std::make_shared(arguments, params); } } -void registerAggregateFunctionCategoricalIV( - AggregateFunctionFactory & factory -) +void registerAggregateFunctionCategoricalIV(AggregateFunctionFactory & factory) { - factory.registerFunction("categoricalInformationValue", createAggregateFunctionCategoricalIV); + AggregateFunctionProperties properties = { .returns_default_when_only_null = true }; + factory.registerFunction("categoricalInformationValue", { createAggregateFunctionCategoricalIV, properties }); } } diff --git a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h b/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h deleted file mode 100644 index 0e0db27cf22..00000000000 --- a/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h +++ /dev/null @@ -1,135 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include - - -namespace DB -{ -struct Settings; - -template -class AggregateFunctionCategoricalIV final : public IAggregateFunctionHelper> -{ -private: - size_t category_count; - -public: - AggregateFunctionCategoricalIV(const DataTypes & arguments_, const Array & params_) : - IAggregateFunctionHelper> {arguments_, params_}, - category_count {arguments_.size() - 1} - { - // notice: argument types has been checked before - } - - String getName() const override - { - return "categoricalInformationValue"; - } - - bool allocatesMemoryInArena() const override { return false; } - - void create(AggregateDataPtr __restrict place) const override - { - memset(place, 0, sizeOfData()); - } - - void destroy(AggregateDataPtr __restrict) const noexcept override - { - // nothing - } - - bool hasTrivialDestructor() const override - { - return true; - } - - size_t sizeOfData() const override - { - return sizeof(T) * (category_count + 1) * 2; - } - - size_t alignOfData() const override - { - return alignof(T); - } - - void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override - { - const auto * y_col = static_cast(columns[category_count]); - bool y = y_col->getData()[row_num]; - - for (size_t i : collections::range(0, category_count)) - { - const auto * x_col = static_cast(columns[i]); - bool x = x_col->getData()[row_num]; - - if (x) - reinterpret_cast(place)[i * 2 + size_t(y)] += 1; - } - - reinterpret_cast(place)[category_count * 2 + size_t(y)] += 1; - } - - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override - { - for (size_t i : collections::range(0, category_count + 1)) - { - reinterpret_cast(place)[i * 2] += reinterpret_cast(rhs)[i * 2]; - reinterpret_cast(place)[i * 2 + 1] += reinterpret_cast(rhs)[i * 2 + 1]; - } - } - - void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override - { - buf.write(place, sizeOfData()); - } - - void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override - { - buf.read(place, sizeOfData()); - } - - DataTypePtr getReturnType() const override - { - return std::make_shared( - std::make_shared>() - ); - } - - void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override /// NOLINT - { - auto & col = static_cast(to); - auto & data_col = static_cast(col.getData()); - auto & offset_col = static_cast( - col.getOffsetsColumn() - ); - - data_col.reserve(data_col.size() + category_count); - - T sum_no = reinterpret_cast(place)[category_count * 2]; - T sum_yes = reinterpret_cast(place)[category_count * 2 + 1]; - - Float64 rev_no = 1. / sum_no; - Float64 rev_yes = 1. / sum_yes; - - for (size_t i : collections::range(0, category_count)) - { - T no = reinterpret_cast(place)[i * 2]; - T yes = reinterpret_cast(place)[i * 2 + 1]; - - data_col.insertValue((no * rev_no - yes * rev_yes) * (log(no * rev_no) - log(yes * rev_yes))); - } - - offset_col.insertValue(data_col.size()); - } -}; - -} diff --git a/src/AggregateFunctions/AggregateFunctionDistinct.h b/src/AggregateFunctions/AggregateFunctionDistinct.h index 5afe104bcc0..482d21363fe 100644 --- a/src/AggregateFunctions/AggregateFunctionDistinct.h +++ b/src/AggregateFunctions/AggregateFunctionDistinct.h @@ -225,6 +225,12 @@ public: nested_func->destroy(getNestedPlace(place)); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + this->data(place).~Data(); + nested_func->destroyUpToState(getNestedPlace(place)); + } + String getName() const override { return nested_func->getName() + "Distinct"; @@ -245,6 +251,21 @@ public: return nested_func->isState(); } + bool isVersioned() const override + { + return nested_func->isVersioned(); + } + + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + + size_t getDefaultVersion() const override + { + return nested_func->getDefaultVersion(); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_func; } }; diff --git a/src/AggregateFunctions/AggregateFunctionForEach.h b/src/AggregateFunctions/AggregateFunctionForEach.h index 064b7b00c86..07713dcb304 100644 --- a/src/AggregateFunctions/AggregateFunctionForEach.h +++ b/src/AggregateFunctions/AggregateFunctionForEach.h @@ -66,6 +66,7 @@ private: if (old_size < new_size) { char * old_state = state.array_of_aggregate_datas; + char * new_state = arena.alignedAlloc( new_size * nested_size_of_data, nested_func->alignOfData()); @@ -134,23 +135,43 @@ public: return nested_func->isVersioned(); } + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + size_t getDefaultVersion() const override { return nested_func->getDefaultVersion(); } - void destroy(AggregateDataPtr __restrict place) const noexcept override + template + void destroyImpl(AggregateDataPtr __restrict place) const noexcept { AggregateFunctionForEachData & state = data(place); char * nested_state = state.array_of_aggregate_datas; for (size_t i = 0; i < state.dynamic_array_size; ++i) { - nested_func->destroy(nested_state); + if constexpr (up_to_state) + nested_func->destroyUpToState(nested_state); + else + nested_func->destroy(nested_state); + nested_state += nested_size_of_data; } } + void destroy(AggregateDataPtr __restrict place) const noexcept override + { + destroyImpl(place); + } + + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + destroyImpl(place); + } + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); diff --git a/src/AggregateFunctions/AggregateFunctionIf.cpp b/src/AggregateFunctions/AggregateFunctionIf.cpp index fa5e6b85a1e..0cf92585b77 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.cpp +++ b/src/AggregateFunctions/AggregateFunctionIf.cpp @@ -278,6 +278,71 @@ public: } } + void addBatchSinglePlace( + size_t row_begin, size_t row_end, AggregateDataPtr __restrict place, const IColumn ** columns, Arena * arena, ssize_t) const final + { + std::unique_ptr final_null_flags = std::make_unique(row_end); + const size_t filter_column_num = number_of_arguments - 1; + + if (is_nullable[filter_column_num]) + { + const ColumnNullable * nullable_column = assert_cast(columns[filter_column_num]); + const IColumn & filter_column = nullable_column->getNestedColumn(); + const UInt8 * filter_null_map = nullable_column->getNullMapColumn().getData().data(); + const UInt8 * filter_values = assert_cast(filter_column).getData().data(); + + for (size_t i = row_begin; i < row_end; i++) + { + final_null_flags[i] = (null_is_skipped && filter_null_map[i]) || !filter_values[i]; + } + } + else + { + const IColumn * filter_column = columns[filter_column_num]; + const UInt8 * filter_values = assert_cast(filter_column)->getData().data(); + for (size_t i = row_begin; i < row_end; i++) + final_null_flags[i] = !filter_values[i]; + } + + const IColumn * nested_columns[number_of_arguments]; + for (size_t arg = 0; arg < number_of_arguments; arg++) + { + if (is_nullable[arg]) + { + const ColumnNullable & nullable_col = assert_cast(*columns[arg]); + if (null_is_skipped && (arg != filter_column_num)) + { + const ColumnUInt8 & nullmap_column = nullable_col.getNullMapColumn(); + const UInt8 * col_null_map = nullmap_column.getData().data(); + for (size_t r = row_begin; r < row_end; r++) + { + final_null_flags[r] |= col_null_map[r]; + } + } + nested_columns[arg] = &nullable_col.getNestedColumn(); + } + else + nested_columns[arg] = columns[arg]; + } + + bool at_least_one = false; + for (size_t i = row_begin; i < row_end; i++) + { + if (!final_null_flags[i]) + { + at_least_one = true; + break; + } + } + + if (at_least_one) + { + this->setFlag(place); + this->nested_function->addBatchSinglePlaceNotNull( + row_begin, row_end, this->nestedPlace(place), nested_columns, final_null_flags.get(), arena, -1); + } + } + #if USE_EMBEDDED_COMPILER void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const DataTypes & arguments_types, const std::vector & argument_values) const override diff --git a/src/AggregateFunctions/AggregateFunctionIf.h b/src/AggregateFunctions/AggregateFunctionIf.h index 18104f94fad..6b0905d6d5e 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.h +++ b/src/AggregateFunctions/AggregateFunctionIf.h @@ -71,6 +71,11 @@ public: return nested_func->isVersioned(); } + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + size_t getDefaultVersion() const override { return nested_func->getDefaultVersion(); @@ -86,6 +91,11 @@ public: nested_func->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + nested_func->destroyUpToState(place); + } + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h index 9ed4b48c281..4cb26fcc8d1 100644 --- a/src/AggregateFunctions/AggregateFunctionMap.h +++ b/src/AggregateFunctions/AggregateFunctionMap.h @@ -84,6 +84,26 @@ private: using Base = IAggregateFunctionDataHelper>; public: + bool isState() const override + { + return nested_func->isState(); + } + + bool isVersioned() const override + { + return nested_func->isVersioned(); + } + + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + + size_t getDefaultVersion() const override + { + return nested_func->getDefaultVersion(); + } + AggregateFunctionMap(AggregateFunctionPtr nested, const DataTypes & types) : Base(types, nested->getParameters()), nested_func(nested) { if (types.empty()) @@ -187,6 +207,32 @@ public: } } + template + void destroyImpl(AggregateDataPtr __restrict place) const noexcept + { + AggregateFunctionMapCombinatorData & state = Base::data(place); + + for (const auto & [key, nested_place] : state.merged_maps) + { + if constexpr (up_to_state) + nested_func->destroyUpToState(nested_place); + else + nested_func->destroy(nested_place); + } + + state.~Data(); + } + + void destroy(AggregateDataPtr __restrict place) const noexcept override + { + destroyImpl(place); + } + + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + destroyImpl(place); + } + void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override { auto & merged_maps = this->data(place).merged_maps; diff --git a/src/AggregateFunctions/AggregateFunctionMerge.h b/src/AggregateFunctions/AggregateFunctionMerge.h index 7bf0f5ea00f..bb2d36eeed1 100644 --- a/src/AggregateFunctions/AggregateFunctionMerge.h +++ b/src/AggregateFunctions/AggregateFunctionMerge.h @@ -80,6 +80,11 @@ public: nested_func->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + nested_func->destroyUpToState(place); + } + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); @@ -126,6 +131,11 @@ public: } AggregateFunctionPtr getNestedFunction() const override { return nested_func; } + + bool isState() const override + { + return nested_func->isState(); + } }; } diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index d579a925f9d..ad633418ec3 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -492,7 +492,7 @@ public: void insertResultInto(IColumn & to) const { if (has()) - assert_cast(to).insertDataWithTerminatingZero(getData(), size); + assert_cast(to).insertData(getData(), size); else assert_cast(to).insertDefault(); } @@ -569,7 +569,7 @@ public: void change(const IColumn & column, size_t row_num, Arena * arena) { - changeImpl(assert_cast(column).getDataAtWithTerminatingZero(row_num), arena); + changeImpl(assert_cast(column).getDataAt(row_num), arena); } void change(const Self & to, Arena * arena) @@ -618,7 +618,7 @@ public: bool changeIfLess(const IColumn & column, size_t row_num, Arena * arena) { - if (!has() || assert_cast(column).getDataAtWithTerminatingZero(row_num) < getStringRef()) + if (!has() || assert_cast(column).getDataAt(row_num) < getStringRef()) { change(column, row_num, arena); return true; @@ -640,7 +640,7 @@ public: bool changeIfGreater(const IColumn & column, size_t row_num, Arena * arena) { - if (!has() || assert_cast(column).getDataAtWithTerminatingZero(row_num) > getStringRef()) + if (!has() || assert_cast(column).getDataAt(row_num) > getStringRef()) { change(column, row_num, arena); return true; @@ -667,7 +667,7 @@ public: bool isEqualTo(const IColumn & column, size_t row_num) const { - return has() && assert_cast(column).getDataAtWithTerminatingZero(row_num) == getStringRef(); + return has() && assert_cast(column).getDataAt(row_num) == getStringRef(); } static bool allocatesMemoryInArena() diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index ca284680800..393fd6f5c19 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -114,6 +114,11 @@ public: nested_function->destroy(nestedPlace(place)); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + nested_function->destroyUpToState(nestedPlace(place)); + } + bool hasTrivialDestructor() const override { return nested_function->hasTrivialDestructor(); @@ -189,6 +194,21 @@ public: return nested_function->isState(); } + bool isVersioned() const override + { + return nested_function->isVersioned(); + } + + size_t getVersionFromRevision(size_t revision) const override + { + return nested_function->getVersionFromRevision(revision); + } + + size_t getDefaultVersion() const override + { + return nested_function->getDefaultVersion(); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } #if USE_EMBEDDED_COMPILER @@ -414,6 +434,115 @@ public: this->nested_function->add(this->nestedPlace(place), nested_columns, row_num, arena); } + void addBatchSinglePlace( + size_t row_begin, + size_t row_end, + AggregateDataPtr __restrict place, + const IColumn ** columns, + Arena * arena, + ssize_t if_argument_pos) const final + { + /// We are going to merge all the flags into a single one to be able to call the nested batching functions + std::vector nullable_filters; + const IColumn * nested_columns[number_of_arguments]; + + std::unique_ptr final_flags = nullptr; + const UInt8 * final_flags_ptr = nullptr; + + if (if_argument_pos >= 0) + { + final_flags = std::make_unique(row_end); + final_flags_ptr = final_flags.get(); + + bool included_elements = 0; + const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); + for (size_t i = row_begin; i < row_end; i++) + { + final_flags[i] = !flags.data()[i]; + included_elements += !!flags.data()[i]; + } + + if (included_elements == 0) + return; + if (included_elements != (row_end - row_begin)) + { + nullable_filters.push_back(final_flags_ptr); + } + } + + for (size_t i = 0; i < number_of_arguments; ++i) + { + if (is_nullable[i]) + { + const ColumnNullable & nullable_col = assert_cast(*columns[i]); + nested_columns[i] = &nullable_col.getNestedColumn(); + if constexpr (null_is_skipped) + { + const ColumnUInt8 & nullmap_column = nullable_col.getNullMapColumn(); + nullable_filters.push_back(nullmap_column.getData().data()); + } + } + else + { + nested_columns[i] = columns[i]; + } + } + + /// We can have 0 nullable filters if we don't skip nulls + if (nullable_filters.size() == 0) + { + this->setFlag(place); + this->nested_function->addBatchSinglePlace(row_begin, row_end, this->nestedPlace(place), nested_columns, arena, -1); + return; + } + + bool found_one = false; + if (nullable_filters.size() == 1) + { + /// We can avoid making copies of the only filter but we still need to check that there is data to be added + final_flags_ptr = nullable_filters[0]; + for (size_t i = row_begin; i < row_end; i++) + { + if (!final_flags_ptr[i]) + { + found_one = true; + break; + } + } + } + else + { + if (!final_flags) + { + final_flags = std::make_unique(row_end); + final_flags_ptr = final_flags.get(); + } + + const size_t filter_start = nullable_filters[0] == final_flags_ptr ? 1 : 0; + for (size_t filter = filter_start; filter < nullable_filters.size(); filter++) + { + for (size_t i = row_begin; i < row_end; i++) + final_flags[i] |= nullable_filters[filter][i]; + } + + for (size_t i = row_begin; i < row_end; i++) + { + if (!final_flags_ptr[i]) + { + found_one = true; + break; + } + } + } + + if (!found_one) + return; // Nothing to do and nothing to mark + + this->setFlag(place); + this->nested_function->addBatchSinglePlaceNotNull( + row_begin, row_end, this->nestedPlace(place), nested_columns, final_flags_ptr, arena, -1); + } + #if USE_EMBEDDED_COMPILER diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.h b/src/AggregateFunctions/AggregateFunctionOrFill.h index 4eca1b4df92..c5a0d60224a 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.h +++ b/src/AggregateFunctions/AggregateFunctionOrFill.h @@ -98,6 +98,11 @@ public: nested_function->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + nested_function->destroyUpToState(place); + } + void add( AggregateDataPtr __restrict place, const IColumn ** columns, diff --git a/src/AggregateFunctions/AggregateFunctionResample.h b/src/AggregateFunctions/AggregateFunctionResample.h index 5d7bf95ceee..471a6820939 100644 --- a/src/AggregateFunctions/AggregateFunctionResample.h +++ b/src/AggregateFunctions/AggregateFunctionResample.h @@ -91,6 +91,21 @@ public: return nested_function->isState(); } + bool isVersioned() const override + { + return nested_function->isVersioned(); + } + + size_t getVersionFromRevision(size_t revision) const override + { + return nested_function->getVersionFromRevision(revision); + } + + size_t getDefaultVersion() const override + { + return nested_function->getDefaultVersion(); + } + bool allocatesMemoryInArena() const override { return nested_function->allocatesMemoryInArena(); @@ -134,6 +149,12 @@ public: nested_function->destroy(place + i * size_of_data); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override + { + for (size_t i = 0; i < total; ++i) + nested_function->destroyUpToState(place + i * size_of_data); + } + void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { Key key; diff --git a/src/AggregateFunctions/AggregateFunctionSimpleState.h b/src/AggregateFunctions/AggregateFunctionSimpleState.h index d63d8b71b8c..f50c86c684e 100644 --- a/src/AggregateFunctions/AggregateFunctionSimpleState.h +++ b/src/AggregateFunctions/AggregateFunctionSimpleState.h @@ -56,10 +56,22 @@ public: return nested_func->getDefaultVersion(); } + bool isState() const override + { + return nested_func->isState(); + } + + size_t getVersionFromRevision(size_t revision) const override + { + return nested_func->getVersionFromRevision(revision); + } + void create(AggregateDataPtr __restrict place) const override { nested_func->create(place); } void destroy(AggregateDataPtr __restrict place) const noexcept override { nested_func->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict place) const noexcept override { nested_func->destroyUpToState(place); } + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); } size_t sizeOfData() const override { return nested_func->sizeOfData(); } diff --git a/src/AggregateFunctions/AggregateFunctionState.h b/src/AggregateFunctions/AggregateFunctionState.h index a598e4838cc..6ab3dbab625 100644 --- a/src/AggregateFunctions/AggregateFunctionState.h +++ b/src/AggregateFunctions/AggregateFunctionState.h @@ -69,6 +69,8 @@ public: nested_func->destroy(place); } + void destroyUpToState(AggregateDataPtr __restrict) const noexcept override {} + bool hasTrivialDestructor() const override { return nested_func->hasTrivialDestructor(); diff --git a/src/AggregateFunctions/AggregateFunctionUniqUpTo.h b/src/AggregateFunctions/AggregateFunctionUniqUpTo.h index aecaecbe4bf..48b4c0f2c68 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqUpTo.h +++ b/src/AggregateFunctions/AggregateFunctionUniqUpTo.h @@ -18,11 +18,6 @@ #include -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Warray-bounds" -#endif - namespace DB { struct Settings; @@ -291,7 +286,3 @@ public: } - -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 87cb1006d36..7a4feebbe0f 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -113,6 +113,17 @@ public: /// Delete data for aggregation. virtual void destroy(AggregateDataPtr __restrict place) const noexcept = 0; + /// Delete all combinator states that were used after combinator -State. + /// For example for uniqArrayStateForEachMap(...) it will destroy + /// states that were created by combinators Map and ForEach. + /// It's needed because ColumnAggregateFunction in the result will be + /// responsible only for destruction of states that were created + /// by aggregate function and all combinators before -State combinator. + virtual void destroyUpToState(AggregateDataPtr __restrict place) const noexcept + { + destroy(place); + } + /// It is not necessary to delete data. virtual bool hasTrivialDestructor() const = 0; @@ -277,8 +288,7 @@ public: Arena * arena) const = 0; /** Insert result of aggregate function into result column with batch size. - * If destroy_place_after_insert is true. Then implementation of this method - * must destroy aggregate place if insert state into result column was successful. + * The implementation of this method will destroy aggregate place up to -State if insert state into result column was successful. * All places that were not inserted must be destroyed if there was exception during insert into result column. */ virtual void insertResultIntoBatch( @@ -287,8 +297,7 @@ public: AggregateDataPtr * places, size_t place_offset, IColumn & to, - Arena * arena, - bool destroy_place_after_insert) const = 0; + Arena * arena) const = 0; /** Destroy batch of aggregate places. */ @@ -612,8 +621,7 @@ public: AggregateDataPtr * places, size_t place_offset, IColumn & to, - Arena * arena, - bool destroy_place_after_insert) const override + Arena * arena) const override { size_t batch_index = row_begin; @@ -622,9 +630,9 @@ public: for (; batch_index < row_end; ++batch_index) { static_cast(this)->insertResultInto(places[batch_index] + place_offset, to, arena); - - if (destroy_place_after_insert) - static_cast(this)->destroy(places[batch_index] + place_offset); + /// For State AggregateFunction ownership of aggregate place is passed to result column after insert, + /// so we need to destroy all states up to state of -State combinator. + static_cast(this)->destroyUpToState(places[batch_index] + place_offset); } } catch (...) diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index b92ba8cfd7b..a64c02e823b 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -165,11 +165,6 @@ public: sorted = false; } -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" -#endif - void write(DB::WriteBuffer & buf) const { size_t size = samples.size(); @@ -193,10 +188,6 @@ public: } } -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif - private: /// We allocate some memory on the stack to avoid allocations when there are many objects with a small number of elements. using Element = std::pair; diff --git a/src/AggregateFunctions/UniquesHashSet.h b/src/AggregateFunctions/UniquesHashSet.h index 8648f6e2500..54503e356c2 100644 --- a/src/AggregateFunctions/UniquesHashSet.h +++ b/src/AggregateFunctions/UniquesHashSet.h @@ -424,14 +424,30 @@ public: alloc(new_size_degree); - for (size_t i = 0; i < m_size; ++i) + if (m_size <= 1) { - HashValue x = 0; - DB::readIntBinary(x, rb); - if (x == 0) - has_zero = true; - else - reinsertImpl(x); + for (size_t i = 0; i < m_size; ++i) + { + HashValue x = 0; + DB::readIntBinary(x, rb); + if (x == 0) + has_zero = true; + else + reinsertImpl(x); + } + } + else + { + auto hs = std::make_unique(m_size); + rb.readStrict(reinterpret_cast(hs.get()), m_size * sizeof(HashValue)); + + for (size_t i = 0; i < m_size; ++i) + { + if (hs[i] == 0) + has_zero = true; + else + reinsertImpl(hs[i]); + } } } @@ -458,11 +474,24 @@ public: resize(new_size_degree); } - for (size_t i = 0; i < rhs_size; ++i) + if (rhs_size <= 1) { - HashValue x = 0; - DB::readIntBinary(x, rb); - insertHash(x); + for (size_t i = 0; i < rhs_size; ++i) + { + HashValue x = 0; + DB::readIntBinary(x, rb); + insertHash(x); + } + } + else + { + auto hs = std::make_unique(rhs_size); + rb.readStrict(reinterpret_cast(hs.get()), rhs_size * sizeof(HashValue)); + + for (size_t i = 0; i < rhs_size; ++i) + { + insertHash(hs[i]); + } } } diff --git a/src/BridgeHelper/CatBoostLibraryBridgeHelper.cpp b/src/BridgeHelper/CatBoostLibraryBridgeHelper.cpp new file mode 100644 index 00000000000..b0ef9b91a28 --- /dev/null +++ b/src/BridgeHelper/CatBoostLibraryBridgeHelper.cpp @@ -0,0 +1,194 @@ +#include "CatBoostLibraryBridgeHelper.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +CatBoostLibraryBridgeHelper::CatBoostLibraryBridgeHelper( + ContextPtr context_, + std::optional model_path_, + std::optional library_path_) + : LibraryBridgeHelper(context_->getGlobalContext()) + , model_path(model_path_) + , library_path(library_path_) +{ +} + +Poco::URI CatBoostLibraryBridgeHelper::getPingURI() const +{ + auto uri = createBaseURI(); + uri.setPath(PING_HANDLER); + return uri; +} + +Poco::URI CatBoostLibraryBridgeHelper::getMainURI() const +{ + auto uri = createBaseURI(); + uri.setPath(MAIN_HANDLER); + return uri; +} + + +Poco::URI CatBoostLibraryBridgeHelper::createRequestURI(const String & method) const +{ + auto uri = getMainURI(); + uri.addQueryParameter("version", std::to_string(LIBRARY_BRIDGE_PROTOCOL_VERSION)); + uri.addQueryParameter("method", method); + return uri; +} + +bool CatBoostLibraryBridgeHelper::bridgeHandShake() +{ + String result; + try + { + ReadWriteBufferFromHTTP buf(getPingURI(), Poco::Net::HTTPRequest::HTTP_GET, {}, http_timeouts, credentials); + readString(result, buf); + } + catch (...) + { + tryLogCurrentException(log); + return false; + } + + if (result != "1") + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {}. Check that bridge and server have the same version.", result); + + return true; +} + +ExternalModelInfos CatBoostLibraryBridgeHelper::listModels() +{ + startBridgeSync(); + + ReadWriteBufferFromHTTP buf( + createRequestURI(CATBOOST_LIST_METHOD), + Poco::Net::HTTPRequest::HTTP_POST, + [](std::ostream &) {}, + http_timeouts, credentials); + + ExternalModelInfos result; + + UInt64 num_rows; + readIntBinary(num_rows, buf); + + for (UInt64 i = 0; i < num_rows; ++i) + { + ExternalModelInfo info; + + readStringBinary(info.model_path, buf); + readStringBinary(info.model_type, buf); + + UInt64 t; + readIntBinary(t, buf); + info.loading_start_time = std::chrono::system_clock::from_time_t(t); + + readIntBinary(t, buf); + info.loading_duration = std::chrono::milliseconds(t); + + result.push_back(info); + } + + return result; +} + +void CatBoostLibraryBridgeHelper::removeModel() +{ + startBridgeSync(); + + assert(model_path); + + ReadWriteBufferFromHTTP buf( + createRequestURI(CATBOOST_REMOVEMODEL_METHOD), + Poco::Net::HTTPRequest::HTTP_POST, + [this](std::ostream & os) + { + os << "model_path=" << escapeForFileName(*model_path); + }, + http_timeouts, credentials); + + String result; + readStringBinary(result, buf); + assert(result == "1"); +} + +void CatBoostLibraryBridgeHelper::removeAllModels() +{ + startBridgeSync(); + + ReadWriteBufferFromHTTP buf( + createRequestURI(CATBOOST_REMOVEALLMODELS_METHOD), + Poco::Net::HTTPRequest::HTTP_POST, + [](std::ostream &){}, + http_timeouts, credentials); + + String result; + readStringBinary(result, buf); + assert(result == "1"); +} + +size_t CatBoostLibraryBridgeHelper::getTreeCount() +{ + startBridgeSync(); + + assert(model_path && library_path); + + ReadWriteBufferFromHTTP buf( + createRequestURI(CATBOOST_GETTREECOUNT_METHOD), + Poco::Net::HTTPRequest::HTTP_POST, + [this](std::ostream & os) + { + os << "library_path=" << escapeForFileName(*library_path) << "&"; + os << "model_path=" << escapeForFileName(*model_path); + }, + http_timeouts, credentials); + + size_t result; + readIntBinary(result, buf); + return result; +} + +ColumnPtr CatBoostLibraryBridgeHelper::evaluate(const ColumnsWithTypeAndName & columns) +{ + startBridgeSync(); + + WriteBufferFromOwnString string_write_buf; + Block block(columns); + NativeWriter serializer(string_write_buf, /*client_revision*/ 0, block); + serializer.write(block); + + assert(model_path); + + ReadWriteBufferFromHTTP buf( + createRequestURI(CATBOOST_LIB_EVALUATE_METHOD), + Poco::Net::HTTPRequest::HTTP_POST, + [this, serialized = string_write_buf.str()](std::ostream & os) + { + os << "model_path=" << escapeForFileName(*model_path) << "&"; + os << "data=" << escapeForFileName(serialized); + }, + http_timeouts, credentials); + + NativeReader deserializer(buf, /*server_revision*/ 0); + Block block_read = deserializer.read(); + + return block_read.getColumns()[0]; +} + +} diff --git a/src/BridgeHelper/CatBoostLibraryBridgeHelper.h b/src/BridgeHelper/CatBoostLibraryBridgeHelper.h new file mode 100644 index 00000000000..91c94143147 --- /dev/null +++ b/src/BridgeHelper/CatBoostLibraryBridgeHelper.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +class CatBoostLibraryBridgeHelper : public LibraryBridgeHelper +{ +public: + static constexpr inline auto PING_HANDLER = "/catboost_ping"; + static constexpr inline auto MAIN_HANDLER = "/catboost_request"; + + explicit CatBoostLibraryBridgeHelper( + ContextPtr context_, + std::optional model_path_ = std::nullopt, + std::optional library_path_ = std::nullopt); + + ExternalModelInfos listModels(); + + void removeModel(); /// requires model_path + void removeAllModels(); + + size_t getTreeCount(); /// requires model_path and library_path + ColumnPtr evaluate(const ColumnsWithTypeAndName & columns); /// requires model_path + +protected: + Poco::URI getPingURI() const override; + + Poco::URI getMainURI() const override; + + bool bridgeHandShake() override; + +private: + static constexpr inline auto CATBOOST_LIST_METHOD = "catboost_list"; + static constexpr inline auto CATBOOST_REMOVEMODEL_METHOD = "catboost_removeModel"; + static constexpr inline auto CATBOOST_REMOVEALLMODELS_METHOD = "catboost_removeAllModels"; + static constexpr inline auto CATBOOST_GETTREECOUNT_METHOD = "catboost_GetTreeCount"; + static constexpr inline auto CATBOOST_LIB_EVALUATE_METHOD = "catboost_libEvaluate"; + + Poco::URI createRequestURI(const String & method) const; + + const std::optional model_path; + const std::optional library_path; +}; + +} diff --git a/src/BridgeHelper/IBridgeHelper.h b/src/BridgeHelper/IBridgeHelper.h index 5068e84f885..a3348c81b68 100644 --- a/src/BridgeHelper/IBridgeHelper.h +++ b/src/BridgeHelper/IBridgeHelper.h @@ -12,8 +12,8 @@ namespace DB { -/// Common base class for XDBC and Library bridge helpers. -/// Contains helper methods to check/start bridge sync. +/// Base class for server-side bridge helpers, e.g. xdbc-bridge and library-bridge. +/// Contains helper methods to check/start bridge sync class IBridgeHelper: protected WithContext { diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 465d4358e91..7d05cbb0681 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1080,6 +1080,20 @@ bool ClientBase::receiveSampleBlock(Block & out, ColumnsDescription & columns_de } +void ClientBase::setInsertionTable(const ASTInsertQuery & insert_query) +{ + if (!global_context->hasInsertionTable() && insert_query.table) + { + String table = insert_query.table->as().shortName(); + if (!table.empty()) + { + String database = insert_query.database ? insert_query.database->as().shortName() : ""; + global_context->setInsertionTable(StorageID(database, table)); + } + } +} + + void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr parsed_query) { auto query = query_to_execute; @@ -1129,6 +1143,8 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars { /// If structure was received (thus, server has not thrown an exception), /// send our data with that structure. + setInsertionTable(parsed_insert_query); + sendData(sample, columns_description, parsed_query); receiveEndOfQuery(); } diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 6b19c1b8e02..278056130fd 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -113,6 +113,8 @@ protected: std::vector & external_tables_arguments, std::vector & hosts_and_ports_arguments) = 0; + void setInsertionTable(const ASTInsertQuery & insert_query); + private: void receiveResult(ASTPtr parsed_query); @@ -176,9 +178,6 @@ protected: bool stderr_is_a_tty = false; /// stderr is a terminal. uint64_t terminal_width = 0; - ServerConnectionPtr connection; - ConnectionParameters connection_parameters; - String format; /// Query results output format. bool select_into_file = false; /// If writing result INTO OUTFILE. It affects progress rendering. bool select_into_file_and_stdout = false; /// If writing result INTO OUTFILE AND STDOUT. It affects progress rendering. @@ -199,6 +198,11 @@ protected: SharedContextHolder shared_context; ContextMutablePtr global_context; + std::optional thread_status; + + ServerConnectionPtr connection; + ConnectionParameters connection_parameters; + /// Buffer that reads from stdin in batch mode. ReadBufferFromFileDescriptor std_in{STDIN_FILENO}; /// Console output. diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index 757927d70bc..3ad9f6ba95c 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -16,7 +16,6 @@ namespace ErrorCodes extern const int ATTEMPT_TO_READ_AFTER_EOF; extern const int NETWORK_ERROR; extern const int SOCKET_TIMEOUT; - extern const int DNS_ERROR; } ConnectionEstablisher::ConnectionEstablisher( @@ -91,7 +90,6 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: catch (const Exception & e) { if (e.code() != ErrorCodes::NETWORK_ERROR && e.code() != ErrorCodes::SOCKET_TIMEOUT - && e.code() != ErrorCodes::DNS_ERROR && e.code() != ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF) throw; diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index b10e24f1ae4..7ac68324915 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -31,9 +31,6 @@ LocalConnection::LocalConnection(ContextPtr context_, bool send_progress_, bool /// Authenticate and create a context to execute queries. session.authenticate("default", "", Poco::Net::SocketAddress{}); session.makeSessionContext(); - - if (!CurrentThread::isInitialized()) - thread_status.emplace(); } LocalConnection::~LocalConnection() diff --git a/src/Client/LocalConnection.h b/src/Client/LocalConnection.h index 7967874d11f..7a1a73006ac 100644 --- a/src/Client/LocalConnection.h +++ b/src/Client/LocalConnection.h @@ -156,7 +156,6 @@ private: String description = "clickhouse-local"; std::optional state; - std::optional thread_status; /// Last "server" packet. std::optional next_packet_type; diff --git a/src/Client/QueryFuzzer.cpp b/src/Client/QueryFuzzer.cpp index 9b404e7c5b7..5e231108bed 100644 --- a/src/Client/QueryFuzzer.cpp +++ b/src/Client/QueryFuzzer.cpp @@ -137,9 +137,41 @@ Field QueryFuzzer::fuzzField(Field field) break; } } - else if (type == Field::Types::Array || type == Field::Types::Tuple) + else if (type == Field::Types::Array) { - auto & arr = field.reinterpret(); + auto & arr = field.get(); + + if (fuzz_rand() % 5 == 0 && !arr.empty()) + { + size_t pos = fuzz_rand() % arr.size(); + arr.erase(arr.begin() + pos); + std::cerr << "erased\n"; + } + + if (fuzz_rand() % 5 == 0) + { + if (!arr.empty()) + { + size_t pos = fuzz_rand() % arr.size(); + arr.insert(arr.begin() + pos, fuzzField(arr[pos])); + std::cerr << fmt::format("inserted (pos {})\n", pos); + } + else + { + arr.insert(arr.begin(), getRandomField(0)); + std::cerr << "inserted (0)\n"; + } + + } + + for (auto & element : arr) + { + element = fuzzField(element); + } + } + else if (type == Field::Types::Tuple) + { + auto & arr = field.get(); if (fuzz_rand() % 5 == 0 && !arr.empty()) { diff --git a/src/Client/Suggest.cpp b/src/Client/Suggest.cpp index f8d41853566..552895e754d 100644 --- a/src/Client/Suggest.cpp +++ b/src/Client/Suggest.cpp @@ -187,9 +187,8 @@ void Suggest::fillWordsFromBlock(const Block & block) Words new_words; new_words.reserve(rows); for (size_t i = 0; i < rows; ++i) - { - new_words.emplace_back(column.getDataAt(i).toString()); - } + new_words.emplace_back(column[i].get()); + addWords(std::move(new_words)); } diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index 61c5ded0594..1a008b035b8 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -162,7 +162,7 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues(MutableColumnPtr colum }; callback(res); - res->forEachSubcolumn(callback); + res->forEachSubcolumnRecursively(callback); for (auto * val : data) func->insertResultInto(val, *res, &column_aggregate_func.createOrGetArena()); diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 7bddfc14707..bb56baf9216 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -151,23 +151,24 @@ void ColumnArray::get(size_t n, Field & res) const StringRef ColumnArray::getDataAt(size_t n) const { + assert(n < size()); + /** Returns the range of memory that covers all elements of the array. * Works for arrays of fixed length values. - * For arrays of strings and arrays of arrays, the resulting chunk of memory may not be one-to-one correspondence with the elements, - * since it contains only the data laid in succession, but not the offsets. */ - size_t offset_of_first_elem = offsetAt(n); - StringRef first = getData().getDataAtWithTerminatingZero(offset_of_first_elem); + /// We are using pointer arithmetic on the addresses of the array elements. + if (!data->isFixedAndContiguous()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getDataAt is not supported for {}", getName()); size_t array_size = sizeAt(n); if (array_size == 0) - return StringRef(first.data, 0); + return StringRef(nullptr, 0); - size_t offset_of_last_elem = getOffsets()[n] - 1; - StringRef last = getData().getDataAtWithTerminatingZero(offset_of_last_elem); + size_t offset_of_first_elem = offsetAt(n); + StringRef first = getData().getDataAt(offset_of_first_elem); - return StringRef(first.data, last.data + last.size - first.data); + return StringRef(first.data, first.size * array_size); } @@ -183,7 +184,7 @@ void ColumnArray::insertData(const char * pos, size_t length) /** Similarly - only for arrays of fixed length values. */ if (!data->isFixedAndContiguous()) - throw Exception("Method insertData is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertData is not supported for {}", getName()); size_t field_size = data->sizeOfValueIfFixed(); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 1d88b2e6a26..4e951ec28b8 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -157,6 +157,14 @@ public: callback(data); } + void forEachSubcolumnRecursively(ColumnCallback callback) override + { + callback(offsets); + offsets->forEachSubcolumnRecursively(callback); + callback(data); + data->forEachSubcolumnRecursively(callback); + } + bool structureEquals(const IColumn & rhs) const override { if (const auto * rhs_concrete = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index 99a230720a4..d70021ba0da 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -81,11 +81,6 @@ public: return data->getDataAt(0); } - StringRef getDataAtWithTerminatingZero(size_t) const override - { - return data->getDataAtWithTerminatingZero(0); - } - UInt64 get64(size_t) const override { return data->get64(0); @@ -240,6 +235,12 @@ public: callback(data); } + void forEachSubcolumnRecursively(ColumnCallback callback) override + { + callback(data); + data->forEachSubcolumnRecursively(callback); + } + bool structureEquals(const IColumn & rhs) const override { if (const auto * rhs_concrete = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index 7cd226c4c11..20bc45eb569 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -59,10 +59,6 @@ public: void get(size_t n, Field & res) const override { getDictionary().get(getIndexes().getUInt(n), res); } StringRef getDataAt(size_t n) const override { return getDictionary().getDataAt(getIndexes().getUInt(n)); } - StringRef getDataAtWithTerminatingZero(size_t n) const override - { - return getDictionary().getDataAtWithTerminatingZero(getIndexes().getUInt(n)); - } bool isDefaultAt(size_t n) const override { return getDictionary().isDefaultAt(getIndexes().getUInt(n)); } UInt64 get64(size_t n) const override { return getDictionary().get64(getIndexes().getUInt(n)); } @@ -177,6 +173,19 @@ public: callback(dictionary.getColumnUniquePtr()); } + void forEachSubcolumnRecursively(ColumnCallback callback) override + { + callback(idx.getPositionsPtr()); + idx.getPositionsPtr()->forEachSubcolumnRecursively(callback); + + /// Column doesn't own dictionary if it's shared. + if (!dictionary.isShared()) + { + callback(dictionary.getColumnUniquePtr()); + dictionary.getColumnUniquePtr()->forEachSubcolumnRecursively(callback); + } + } + bool structureEquals(const IColumn & rhs) const override { if (const auto * rhs_low_cardinality = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index 9956ba19455..7377707bdb2 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -278,6 +278,12 @@ void ColumnMap::forEachSubcolumn(ColumnCallback callback) callback(nested); } +void ColumnMap::forEachSubcolumnRecursively(ColumnCallback callback) +{ + callback(nested); + nested->forEachSubcolumnRecursively(callback); +} + bool ColumnMap::structureEquals(const IColumn & rhs) const { if (const auto * rhs_map = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h index 95838e70d10..a3e171008ff 100644 --- a/src/Columns/ColumnMap.h +++ b/src/Columns/ColumnMap.h @@ -89,6 +89,7 @@ public: size_t allocatedBytes() const override; void protect() override; void forEachSubcolumn(ColumnCallback callback) override; + void forEachSubcolumnRecursively(ColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; double getRatioOfDefaultRows(double sample_ratio) const override; void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override; diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 064becbf2f4..bb4c881c54c 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -136,6 +136,14 @@ public: callback(null_map); } + void forEachSubcolumnRecursively(ColumnCallback callback) override + { + callback(nested_column); + nested_column->forEachSubcolumnRecursively(callback); + callback(null_map); + null_map->forEachSubcolumnRecursively(callback); + } + bool structureEquals(const IColumn & rhs) const override { if (const auto * rhs_nullable = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 5b72b838b99..86586559ff7 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -671,6 +671,18 @@ void ColumnObject::forEachSubcolumn(ColumnCallback callback) callback(part); } +void ColumnObject::forEachSubcolumnRecursively(ColumnCallback callback) +{ + for (auto & entry : subcolumns) + { + for (auto & part : entry->data.data) + { + callback(part); + part->forEachSubcolumnRecursively(callback); + } + } +} + void ColumnObject::insert(const Field & field) { const auto & object = field.get(); diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index 07099307258..f32356fed6e 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -211,6 +211,7 @@ public: size_t byteSize() const override; size_t allocatedBytes() const override; void forEachSubcolumn(ColumnCallback callback) override; + void forEachSubcolumnRecursively(ColumnCallback callback) override; void insert(const Field & field) override; void insertDefault() override; void insertFrom(const IColumn & src, size_t n) override; diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp index c6f0c753d64..0c3f8d11adc 100644 --- a/src/Columns/ColumnSparse.cpp +++ b/src/Columns/ColumnSparse.cpp @@ -750,6 +750,14 @@ void ColumnSparse::forEachSubcolumn(ColumnCallback callback) callback(offsets); } +void ColumnSparse::forEachSubcolumnRecursively(ColumnCallback callback) +{ + callback(values); + values->forEachSubcolumnRecursively(callback); + callback(offsets); + offsets->forEachSubcolumnRecursively(callback); +} + const IColumn::Offsets & ColumnSparse::getOffsetsData() const { return assert_cast(*offsets).getData(); diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h index 04d8f6ae9b8..5814cd77637 100644 --- a/src/Columns/ColumnSparse.h +++ b/src/Columns/ColumnSparse.h @@ -140,6 +140,7 @@ public: ColumnPtr compress() const override; void forEachSubcolumn(ColumnCallback callback) override; + void forEachSubcolumnRecursively(ColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 361b792df55..863b080c588 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -108,24 +108,12 @@ public: return StringRef(&chars[offsetAt(n)], sizeAt(n) - 1); } - StringRef getDataAtWithTerminatingZero(size_t n) const override - { - assert(n < size()); - return StringRef(&chars[offsetAt(n)], sizeAt(n)); - } - bool isDefaultAt(size_t n) const override { assert(n < size()); return sizeAt(n) == 1; } -/// Suppress gcc 7.3.1 warning: '*((void*)& +8)' may be used uninitialized in this function -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - void insert(const Field & x) override { const String & s = x.get(); @@ -138,10 +126,6 @@ public: offsets.push_back(new_size); } -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif - void insertFrom(const IColumn & src_, size_t n) override { const ColumnString & src = assert_cast(src_); @@ -177,17 +161,6 @@ public: offsets.push_back(new_size); } - /// Like getData, but inserting data should be zero-ending (i.e. length is 1 byte greater than real string size). - void insertDataWithTerminatingZero(const char * pos, size_t length) - { - const size_t old_size = chars.size(); - const size_t new_size = old_size + length; - - chars.resize(new_size); - memcpy(chars.data() + old_size, pos, length); - offsets.push_back(new_size); - } - void popBack(size_t n) override { size_t nested_n = offsets.back() - offsetAt(offsets.size() - n); diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index f023798e589..3577b6dee28 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -501,6 +501,15 @@ void ColumnTuple::forEachSubcolumn(ColumnCallback callback) callback(column); } +void ColumnTuple::forEachSubcolumnRecursively(ColumnCallback callback) +{ + for (auto & column : columns) + { + callback(column); + column->forEachSubcolumnRecursively(callback); + } +} + bool ColumnTuple::structureEquals(const IColumn & rhs) const { if (const auto * rhs_tuple = typeid_cast(&rhs)) diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index b1de8df74a9..385de7db1e7 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -97,6 +97,7 @@ public: size_t allocatedBytes() const override; void protect() override; void forEachSubcolumn(ColumnCallback callback) override; + void forEachSubcolumnRecursively(ColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; bool isCollationSupported() const override; ColumnPtr compress() const override; diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index d3ab87410f3..f53af42c927 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -1,4 +1,5 @@ #pragma once + #include #include #include @@ -7,16 +8,17 @@ #include #include #include +#include #include #include #include #include -#include +#include +#include #include -#include "Columns/ColumnConst.h" namespace DB @@ -70,10 +72,6 @@ public: void get(size_t n, Field & res) const override { getNestedColumn()->get(n, res); } bool isDefaultAt(size_t n) const override { return n == 0; } StringRef getDataAt(size_t n) const override { return getNestedColumn()->getDataAt(n); } - StringRef getDataAtWithTerminatingZero(size_t n) const override - { - return getNestedColumn()->getDataAtWithTerminatingZero(n); - } UInt64 get64(size_t n) const override { return getNestedColumn()->get64(n); } UInt64 getUInt(size_t n) const override { return getNestedColumn()->getUInt(n); } Int64 getInt(size_t n) const override { return getNestedColumn()->getInt(n); } @@ -115,6 +113,15 @@ public: nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask); } + void forEachSubcolumnRecursively(IColumn::ColumnCallback callback) override + { + callback(column_holder); + column_holder->forEachSubcolumnRecursively(callback); + reverse_index.setColumn(getRawColumnPtr()); + if (is_nullable) + nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask); + } + bool structureEquals(const IColumn & rhs) const override { if (auto rhs_concrete = typeid_cast(&rhs)) @@ -309,17 +316,52 @@ size_t ColumnUnique::getNullValueIndex() const return 0; } + +namespace +{ + class FieldVisitorGetData : public StaticVisitor<> + { + public: + StringRef res; + + [[noreturn]] static void throwUnsupported() + { + throw Exception("Unsupported field type", ErrorCodes::LOGICAL_ERROR); + } + + [[noreturn]] void operator() (const Null &) { throwUnsupported(); } + [[noreturn]] void operator() (const Array &) { throwUnsupported(); } + [[noreturn]] void operator() (const Tuple &) { throwUnsupported(); } + [[noreturn]] void operator() (const Map &) { throwUnsupported(); } + [[noreturn]] void operator() (const Object &) { throwUnsupported(); } + [[noreturn]] void operator() (const AggregateFunctionStateData &) { throwUnsupported(); } + void operator() (const String & x) { res = {x.data(), x.size()}; } + void operator() (const UInt64 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const UInt128 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const UInt256 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const Int64 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const Int128 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const Int256 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const UUID & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const Float64 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + void operator() (const bool & x) { res = {reinterpret_cast(&x), sizeof(x)}; } + }; +} + + template size_t ColumnUnique::uniqueInsert(const Field & x) { if (x.isNull()) return getNullValueIndex(); - if (valuesHaveFixedSize()) - return uniqueInsertData(&x.reinterpret(), size_of_value_if_fixed); - - const auto & val = x.get(); - return uniqueInsertData(val.data(), val.size()); + FieldVisitorGetData visitor; + applyVisitor(visitor, x); + return uniqueInsertData(visitor.res.data, visitor.res.size); } template diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 974925d247e..380eb36f87b 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -106,13 +106,6 @@ public: /// Is used to optimize some computations (in aggregation, for example). [[nodiscard]] virtual StringRef getDataAt(size_t n) const = 0; - /// Like getData, but has special behavior for columns that contain variable-length strings. - /// Returns zero-ending memory chunk (i.e. its size is 1 byte longer). - [[nodiscard]] virtual StringRef getDataAtWithTerminatingZero(size_t n) const - { - return getDataAt(n); - } - /// If column stores integers, it returns n-th element transformed to UInt64 using static_cast. /// If column stores floating point numbers, bits of n-th elements are copied to lower bits of UInt64, the remaining bits are zeros. /// Is used to optimize some computations (in aggregation, for example). @@ -421,6 +414,9 @@ public: using ColumnCallback = std::function; virtual void forEachSubcolumn(ColumnCallback) {} + /// Similar to forEachSubcolumn but it also do recursive calls. + virtual void forEachSubcolumnRecursively(ColumnCallback) {} + /// Columns have equal structure. /// If true - you can use "compareAt", "insertFrom", etc. methods. [[nodiscard]] virtual bool structureEquals(const IColumn &) const diff --git a/src/Columns/tests/gtest_column_object.cpp b/src/Columns/tests/gtest_column_object.cpp index e1ad949f6a8..f9b6ff16b71 100644 --- a/src/Columns/tests/gtest_column_object.cpp +++ b/src/Columns/tests/gtest_column_object.cpp @@ -1,8 +1,11 @@ #include #include +#include +#include #include #include #include +#include #include #include @@ -118,3 +121,36 @@ TEST(ColumnObject, InsertRangeFrom) checkFieldsAreEqual(subcolumn_dst, fields_dst); } } + +TEST(ColumnObject, Unflatten) +{ + auto check_empty_tuple = [](const auto & type, const auto & column) + { + const auto & type_tuple = assert_cast(*type); + const auto & column_tuple = assert_cast(*column); + + ASSERT_EQ(type_tuple.getElements().size(), 1); + ASSERT_EQ(type_tuple.getElements()[0]->getName(), "UInt8"); + ASSERT_EQ(type_tuple.getElementNames()[0], ColumnObject::COLUMN_NAME_DUMMY); + + ASSERT_EQ(column_tuple.getColumns().size(), 1); + ASSERT_EQ(column_tuple.getColumns()[0]->getName(), "UInt8"); + }; + + { + auto column_object = ColumnObject::create(false); + auto [column, type] = unflattenObjectToTuple(*column_object); + + check_empty_tuple(type, column); + ASSERT_EQ(column->size(), 0); + } + + { + auto column_object = ColumnObject::create(false); + column_object->insertManyDefaults(5); + auto [column, type] = unflattenObjectToTuple(*column_object); + + check_empty_tuple(type, column); + ASSERT_EQ(column->size(), 5); + } +} diff --git a/src/Common/Allocator.h b/src/Common/Allocator.h index 06ccbed4064..c348eaea006 100644 --- a/src/Common/Allocator.h +++ b/src/Common/Allocator.h @@ -281,14 +281,6 @@ private: #endif }; -/** When using AllocatorWithStackMemory, located on the stack, - * GCC 4.9 mistakenly assumes that we can call `free` from a pointer to the stack. - * In fact, the combination of conditions inside AllocatorWithStackMemory does not allow this. - */ -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfree-nonheap-object" -#endif /** Allocator with optimization to place small memory ranges in automatic memory. */ @@ -366,7 +358,3 @@ extern template class Allocator; extern template class Allocator; extern template class Allocator; extern template class Allocator; - -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif diff --git a/src/Common/ArrayCache.h b/src/Common/ArrayCache.h index f01ff94e38b..79aeddb09df 100644 --- a/src/Common/ArrayCache.h +++ b/src/Common/ArrayCache.h @@ -722,5 +722,3 @@ public: return res; } }; - -template constexpr size_t ArrayCache::min_chunk_size; diff --git a/src/Common/Base64.cpp b/src/Common/Base64.cpp new file mode 100644 index 00000000000..74ce979b5b1 --- /dev/null +++ b/src/Common/Base64.cpp @@ -0,0 +1,33 @@ +#include + +#include +#include +#include +#include + +#include + +namespace DB +{ + +std::string base64Encode(const std::string & decoded, bool url_encoding) +{ + std::ostringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + ostr.exceptions(std::ios::failbit); + Poco::Base64Encoder encoder(ostr, url_encoding ? Poco::BASE64_URL_ENCODING : 0); + encoder.rdbuf()->setLineLength(0); + encoder << decoded; + encoder.close(); + return ostr.str(); +} + +std::string base64Decode(const std::string & encoded, bool url_encoding) +{ + std::string decoded; + Poco::MemoryInputStream istr(encoded.data(), encoded.size()); + Poco::Base64Decoder decoder(istr, url_encoding ? Poco::BASE64_URL_ENCODING : 0); + Poco::StreamCopier::copyToString(decoder, decoded); + return decoded; +} + +} diff --git a/src/Common/Base64.h b/src/Common/Base64.h new file mode 100644 index 00000000000..963d3acb48f --- /dev/null +++ b/src/Common/Base64.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace DB +{ + +std::string base64Encode(const std::string & decoded, bool url_encoding = false); + +std::string base64Decode(const std::string & encoded, bool url_encoding = false); + +} diff --git a/src/Common/DateLUTImpl.h b/src/Common/DateLUTImpl.h index f5504749684..564d09aff6e 100644 --- a/src/Common/DateLUTImpl.h +++ b/src/Common/DateLUTImpl.h @@ -29,13 +29,6 @@ #define DATE_LUT_ADD ((1970 - DATE_LUT_MIN_YEAR) * 366L * 86400) -#if defined(__PPC__) -#if !defined(__clang__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif -#endif - - /// Flags for toYearWeek() function. enum class WeekModeFlag : UInt8 { @@ -1445,9 +1438,3 @@ public: return s; } }; - -#if defined(__PPC__) -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif -#endif diff --git a/src/Common/Dwarf.cpp b/src/Common/Dwarf.cpp index c8ffe7d46a8..a912c49d2ae 100644 --- a/src/Common/Dwarf.cpp +++ b/src/Common/Dwarf.cpp @@ -26,6 +26,7 @@ #include #define DW_CHILDREN_no 0 + #define DW_FORM_addr 1 #define DW_FORM_block1 0x0a #define DW_FORM_block2 3 @@ -51,6 +52,25 @@ #define DW_FORM_string 0x08 #define DW_FORM_strp 0x0e #define DW_FORM_indirect 0x16 +#define DW_FORM_strx 0x1a +#define DW_FORM_addrx 0x1b +#define DW_FORM_ref_sup4 0x1c +#define DW_FORM_strp_sup 0x1d +#define DW_FORM_data16 0x1e +#define DW_FORM_line_strp 0x1f +#define DW_FORM_implicit_const 0x21 +#define DW_FORM_rnglistx 0x23 +#define DW_FORM_loclistx 0x22 +#define DW_FORM_ref_sup8 0x24 +#define DW_FORM_strx1 0x25 +#define DW_FORM_strx2 0x26 +#define DW_FORM_strx3 0x27 +#define DW_FORM_strx4 0x28 +#define DW_FORM_addrx1 0x29 +#define DW_FORM_addrx2 0x2a +#define DW_FORM_addrx3 0x2b +#define DW_FORM_addrx4 0x2c + #define DW_TAG_compile_unit 0x11 #define DW_TAG_subprogram 0x2e #define DW_TAG_try_block 0x32 @@ -58,6 +78,7 @@ #define DW_TAG_entry_point 0x03 #define DW_TAG_common_block 0x1a #define DW_TAG_lexical_block 0x0b + #define DW_AT_stmt_list 0x10 #define DW_AT_comp_dir 0x1b #define DW_AT_name 0x03 @@ -70,6 +91,13 @@ #define DW_AT_call_file 0x58 #define DW_AT_linkage_name 0x6e #define DW_AT_specification 0x47 +#define DW_AT_str_offsets_base 0x72 +#define DW_AT_addr_base 0x73 +#define DW_AT_rnglists_base 0x74 +#define DW_AT_loclists_base 0x8c +#define DW_AT_GNU_ranges_base 0x2132 +#define DW_AT_GNU_addr_base 0x2133 + #define DW_LNE_define_file 0x03 #define DW_LNS_copy 0x01 #define DW_LNS_advance_pc 0x02 @@ -87,6 +115,21 @@ #define DW_LNE_set_address 0x02 #define DW_LNE_set_discriminator 0x04 +#define DW_LNCT_path 0x1 +#define DW_LNCT_directory_index 0x2 +#define DW_LNCT_timestamp 0x3 +#define DW_LNCT_size 0x4 +#define DW_LNCT_MD5 0x5 + +#define DW_RLE_end_of_list 0x0 +#define DW_RLE_base_addressx 0x1 +#define DW_RLE_startx_endx 0x2 +#define DW_RLE_startx_length 0x3 +#define DW_RLE_offset_pair 0x4 +#define DW_RLE_base_address 0x5 +#define DW_RLE_start_end 0x6 +#define DW_RLE_start_length 0x7 + namespace DB { @@ -97,9 +140,31 @@ namespace ErrorCodes } -Dwarf::Dwarf(const std::shared_ptr & elf) : elf_(elf) +Dwarf::Dwarf(const std::shared_ptr & elf) + : elf_(elf) + , abbrev_(getSection(".debug_abbrev")) + , addr_(getSection(".debug_addr")) + , aranges_(getSection(".debug_aranges")) + , info_(getSection(".debug_info")) + , line_(getSection(".debug_line")) + , line_str_(getSection(".debug_line_str")) + , loclists_(getSection(".debug_loclists")) + , ranges_(getSection(".debug_ranges")) + , rnglists_(getSection(".debug_rnglists")) + , str_(getSection(".debug_str")) + , str_offsets_(getSection(".debug_str_offsets")) { - init(); + // Optional sections: + // - debugAranges_: for fast address range lookup. + // If missing .debug_info can be used - but it's much slower (linear + // scan). + // - debugRanges_ (DWARF 4) / debugRnglists_ (DWARF 5): non-contiguous + // address ranges of debugging information entries. + // Used for inline function address lookup. + if (info_.empty() || abbrev_.empty() || line_.empty() || str_.empty()) + { + elf_ = nullptr; + } } Dwarf::Section::Section(std::string_view d) : is64_bit(false), data(d) @@ -107,7 +172,7 @@ Dwarf::Section::Section(std::string_view d) : is64_bit(false), data(d) } -#define SAFE_CHECK(cond, message) do { if (!(cond)) throw Exception(message, ErrorCodes::CANNOT_PARSE_DWARF); } while (false) +#define SAFE_CHECK(cond, ...) do { if (!(cond)) throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, __VA_ARGS__); } while (false) namespace @@ -124,13 +189,24 @@ template requires std::is_trivial_v && std::is_standard_layout_v T read(std::string_view & sp) { - SAFE_CHECK(sp.size() >= sizeof(T), fmt::format("underflow: expected bytes {}, got bytes {}", sizeof(T), sp.size())); + SAFE_CHECK(sp.size() >= sizeof(T), "underflow: expected bytes {}, got bytes {}", sizeof(T), sp.size()); T x; memcpy(&x, sp.data(), sizeof(T)); sp.remove_prefix(sizeof(T)); return x; } +// Read (bitwise) an unsigned number of N bytes (N in 1, 2, 3, 4). +template +uint64_t readU64(std::string_view & sp) +{ + SAFE_CHECK(sp.size() >= N, "underflow"); + uint64_t x = 0; + memcpy(&x, sp.data(), N); + sp.remove_prefix(N); + return x; +} + // Read ULEB (unsigned) varint value; algorithm from the DWARF spec uint64_t readULEB(std::string_view & sp, uint8_t & shift, uint8_t & val) { @@ -168,9 +244,9 @@ int64_t readSLEB(std::string_view & sp) } // Read a value of "section offset" type, which may be 4 or 8 bytes -uint64_t readOffset(std::string_view & sp, bool is64Bit) +uint64_t readOffset(std::string_view & sp, bool is64_bit) { - return is64Bit ? read(sp) : read(sp); + return is64_bit ? read(sp) : read(sp); } // Read "len" bytes @@ -192,6 +268,15 @@ std::string_view readNullTerminated(std::string_view & sp) return ret; } +// Get a string from the section +std::string_view getStringFromStringSection(std::string_view section, uint64_t offset) +{ + SAFE_CHECK(offset < section.size(), "invalid section offset"); + std::string_view sp(section); + sp.remove_prefix(offset); + return readNullTerminated(sp); +} + // Skip over padding until sp.data() - start is a multiple of alignment void skipPadding(std::string_view & sp, const char * start, size_t alignment) { @@ -359,38 +444,18 @@ bool Dwarf::Section::next(std::string_view & chunk) return true; } -bool Dwarf::getSection(const char * name, std::string_view * section) const +std::string_view Dwarf::getSection(const char * name) const { std::optional elf_section = elf_->findSectionByName(name); if (!elf_section) - return false; + return {}; #ifdef SHF_COMPRESSED if (elf_section->header.sh_flags & SHF_COMPRESSED) - return false; + return {}; #endif - *section = { elf_section->begin(), elf_section->size()}; - return true; -} - -void Dwarf::init() -{ - // Make sure that all .debug_* sections exist - if (!getSection(".debug_info", &info_) - || !getSection(".debug_abbrev", &abbrev_) - || !getSection(".debug_line", &line_) - || !getSection(".debug_str", &strings_)) - { - elf_.reset(); - return; - } - - // Optional: fast address range lookup. If missing .debug_info can - // be used - but it's much slower (linear scan). - getSection(".debug_aranges", &aranges_); - - getSection(".debug_ranges", &ranges_); + return { elf_section->begin(), elf_section->size()}; } // static @@ -473,7 +538,7 @@ size_t Dwarf::forEachAttribute(const CompilationUnit & cu, const Die & die, std: auto values = std::string_view{info_.data() + die.offset + die.attr_offset, cu.offset + cu.size - die.offset - die.attr_offset}; while (auto spec = readAttributeSpec(attrs)) { - auto attr = readAttribute(die, spec, values); + auto attr = readAttribute(cu, die, spec, values); if (!f(attr)) { return static_cast(-1); @@ -482,8 +547,49 @@ size_t Dwarf::forEachAttribute(const CompilationUnit & cu, const Die & die, std: return values.data() - info_.data(); } -Dwarf::Attribute Dwarf::readAttribute(const Die & die, AttributeSpec spec, std::string_view & info) const +Dwarf::Attribute Dwarf::readAttribute(const CompilationUnit & cu, + const Die & die, + AttributeSpec spec, + std::string_view & info) const { + // DWARF 5 introduces new FORMs whose values are relative to some base attrs: + // DW_AT_str_offsets_base, DW_AT_rnglists_base, DW_AT_addr_base. + // Debug Fission DWARF 4 uses GNU DW_AT_GNU_ranges_base & DW_AT_GNU_addr_base. + // + // The order in which attributes appear in a CU is not defined. + // The DW_AT_*_base attrs may appear after attributes that need them. + // The DW_AT_*_base attrs are CU specific; so we read them just after + // reading the CU header. During this first pass return empty values + // when encountering a FORM that depends on DW_AT_*_base. + auto get_string_using_offset_table = [&](uint64_t index) + { + if (!cu.str_offsets_base.has_value()) + { + return std::string_view(); + } + // DWARF 5: 7.26 String Offsets Table + // The DW_AT_str_offsets_base attribute points to the first entry following + // the header. The entries are indexed sequentially from this base entry, + // starting from 0. + auto sp = str_offsets_.substr(*cu.str_offsets_base + index * (cu.is64Bit ? sizeof(uint64_t) : sizeof(uint32_t))); + uint64_t str_offset = readOffset(sp, cu.is64Bit); + return getStringFromStringSection(str_, str_offset); + }; + + auto read_debug_addr = [&](uint64_t index) + { + if (!cu.addr_base.has_value()) + { + return uint64_t(0); + } + // DWARF 5: 7.27 Address Table + // The DW_AT_addr_base attribute points to the first entry following the + // header. The entries are indexed sequentially from this base entry, + // starting from 0. + auto sp = addr_.substr(*cu.addr_base + index * sizeof(uint64_t)); + return read(sp); + }; + switch (spec.form) { case DW_FORM_addr: @@ -517,7 +623,7 @@ Dwarf::Attribute Dwarf::readAttribute(const Die & die, AttributeSpec spec, std:: case DW_FORM_ref_sig8: return {spec, die, read(info)}; case DW_FORM_sdata: - return {spec, die, uint64_t(readSLEB(info))}; + return {spec, die, static_cast(readSLEB(info))}; case DW_FORM_udata: [[fallthrough]]; case DW_FORM_ref_udata: @@ -525,7 +631,7 @@ Dwarf::Attribute Dwarf::readAttribute(const Die & die, AttributeSpec spec, std:: case DW_FORM_flag: return {spec, die, read(info)}; case DW_FORM_flag_present: - return {spec, die, 1u}; + return {spec, die, 1ULL}; case DW_FORM_sec_offset: [[fallthrough]]; case DW_FORM_ref_addr: @@ -533,49 +639,215 @@ Dwarf::Attribute Dwarf::readAttribute(const Die & die, AttributeSpec spec, std:: case DW_FORM_string: return {spec, die, readNullTerminated(info)}; case DW_FORM_strp: - return {spec, die, getStringFromStringSection(readOffset(info, die.is64Bit))}; + return {spec, die, getStringFromStringSection(str_, readOffset(info, die.is64Bit))}; case DW_FORM_indirect: // form is explicitly specified // Update spec with the actual FORM. spec.form = readULEB(info); - return readAttribute(die, spec, info); + return readAttribute(cu, die, spec, info); + + // DWARF 5: + case DW_FORM_implicit_const: // form is explicitly specified + // For attributes with this form, the attribute specification contains a + // third part, which is a signed LEB128 number. The value of this number + // is used as the value of the attribute, and no value is stored in the + // .debug_info section. + return {spec, die, static_cast(spec.implicitConst)}; + + case DW_FORM_addrx: + return {spec, die, read_debug_addr(readULEB(info))}; + case DW_FORM_addrx1: + return {spec, die, read_debug_addr(readU64<1>(info))}; + case DW_FORM_addrx2: + return {spec, die, read_debug_addr(readU64<2>(info))}; + case DW_FORM_addrx3: + return {spec, die, read_debug_addr(readU64<3>(info))}; + case DW_FORM_addrx4: + return {spec, die, read_debug_addr(readU64<4>(info))}; + + case DW_FORM_line_strp: + return {spec, die, getStringFromStringSection(line_str_, readOffset(info, die.is64Bit))}; + + case DW_FORM_strx: + return {spec, die, get_string_using_offset_table(readULEB(info))}; + case DW_FORM_strx1: + return {spec, die, get_string_using_offset_table(readU64<1>(info))}; + case DW_FORM_strx2: + return {spec, die, get_string_using_offset_table(readU64<2>(info))}; + case DW_FORM_strx3: + return {spec, die, get_string_using_offset_table(readU64<3>(info))}; + case DW_FORM_strx4: + return {spec, die, get_string_using_offset_table(readU64<4>(info))}; + + case DW_FORM_rnglistx: { + auto index = readULEB(info); + if (!cu.rnglists_base.has_value()) + { + return {spec, die, 0ULL}; + } + const uint64_t offset_size = cu.is64Bit ? sizeof(uint64_t) : sizeof(uint32_t); + auto sp = rnglists_.substr(*cu.rnglists_base + index * offset_size); + auto offset = readOffset(sp, cu.is64Bit); + return {spec, die, *cu.rnglists_base + offset}; + } + + case DW_FORM_loclistx: { + auto index = readULEB(info); + if (!cu.loclists_base.has_value()) + { + return {spec, die, 0ULL}; + } + const uint64_t offset_size = cu.is64Bit ? sizeof(uint64_t) : sizeof(uint32_t); + auto sp = loclists_.substr(*cu.loclists_base + index * offset_size); + auto offset = readOffset(sp, cu.is64Bit); + return {spec, die, *cu.loclists_base + offset}; + } + + case DW_FORM_data16: + return {spec, die, readBytes(info, 16)}; + + case DW_FORM_ref_sup4: + case DW_FORM_ref_sup8: + case DW_FORM_strp_sup: + SAFE_CHECK(false, "Unexpected DWARF5 supplimentary object files"); + default: SAFE_CHECK(false, "invalid attribute form"); } - - return {spec, die, 0u}; + return {spec, die, 0ULL}; } // static Dwarf::AttributeSpec Dwarf::readAttributeSpec(std::string_view & sp) { - return {readULEB(sp), readULEB(sp)}; + Dwarf::AttributeSpec spec; + spec.name = readULEB(sp); + spec.form = readULEB(sp); + if (spec.form == DW_FORM_implicit_const) + { + spec.implicitConst = readSLEB(sp); + } + return spec; } -// static -Dwarf::CompilationUnit Dwarf::getCompilationUnit(std::string_view info, uint64_t offset) +Dwarf::CompilationUnit Dwarf::getCompilationUnit(uint64_t offset) const { - SAFE_CHECK(offset < info.size(), "unexpected offset"); + // SAFE_CHECK(offset < info_.size(), "unexpected offset"); CompilationUnit cu; - std::string_view chunk(info); + std::string_view chunk(info_); cu.offset = offset; chunk.remove_prefix(offset); + // 1) unit_length auto initial_length = read(chunk); cu.is64Bit = (initial_length == uint32_t(-1)); cu.size = cu.is64Bit ? read(chunk) : initial_length; SAFE_CHECK(cu.size <= chunk.size(), "invalid chunk size"); cu.size += cu.is64Bit ? 12 : 4; + // 2) version cu.version = read(chunk); - SAFE_CHECK(cu.version >= 2 && cu.version <= 4, "invalid info version"); - cu.abbrev_offset = readOffset(chunk, cu.is64Bit); - cu.addr_size = read(chunk); - SAFE_CHECK(cu.addr_size == sizeof(uintptr_t), "invalid address size"); + SAFE_CHECK(cu.version >= 2 && cu.version <= 5, "invalid info version"); - cu.first_die = chunk.data() - info.data(); + if (cu.version == 5) + { + // DWARF5: 7.5.1.1 Full and Partial Compilation Unit Headers + // 3) unit_type (new DWARF 5) + cu.unit_type = read(chunk); + if (cu.unit_type != DW_UT_compile && cu.unit_type != DW_UT_skeleton) + { + return cu; + } + // 4) address_size + cu.addr_size = read(chunk); + SAFE_CHECK(cu.addr_size == sizeof(uintptr_t), "invalid address size"); + + // 5) debug_abbrev_offset + cu.abbrev_offset = readOffset(chunk, cu.is64Bit); + + if (cu.unit_type == DW_UT_skeleton) + { + // 6) dwo_id + read(chunk); + } + } + else + { + // DWARF4 has a single type of unit in .debug_info + cu.unit_type = DW_UT_compile; + // 3) debug_abbrev_offset + cu.abbrev_offset = readOffset(chunk, cu.is64Bit); + // 4) address_size + cu.addr_size = read(chunk); + SAFE_CHECK(cu.addr_size == sizeof(uintptr_t), "invalid address size"); + } + cu.first_die = chunk.data() - info_.data(); + if (cu.version < 5) + { + return cu; + } + + Die die = getDieAtOffset(cu, cu.first_die); + if (die.abbr.tag != DW_TAG_compile_unit) + { + return cu; + } + + // Read the DW_AT_*_base attributes. + // Attributes which use FORMs relative to these base attrs + // will not have valid values during this first pass! + forEachAttribute( + cu, + die, + [&](const Attribute & attr) + { + switch (attr.spec.name) + { + case DW_AT_addr_base: + case DW_AT_GNU_addr_base: + cu.addr_base = std::get(attr.attr_value); + break; + case DW_AT_loclists_base: + cu.loclists_base = std::get(attr.attr_value); + break; + case DW_AT_rnglists_base: + case DW_AT_GNU_ranges_base: + cu.rnglists_base = std::get(attr.attr_value); + break; + case DW_AT_str_offsets_base: + cu.str_offsets_base = std::get(attr.attr_value); + break; + } + return true; // continue forEachAttribute + }); return cu; } +// Finds the Compilation Unit starting at offset. +Dwarf::CompilationUnit Dwarf::findCompilationUnit(uint64_t targetOffset) const +{ + // SAFE_CHECK(targetOffset < info_.size(), "unexpected target address"); + uint64_t offset = 0; + while (offset < info_.size()) + { + std::string_view chunk(info_); + chunk.remove_prefix(offset); + + auto initial_length = read(chunk); + auto is64_bit = (initial_length == static_cast(-1)); + auto size = is64_bit ? read(chunk) : initial_length; + SAFE_CHECK(size <= chunk.size(), "invalid chunk size"); + size += is64_bit ? 12 : 4; + + if (offset + size > targetOffset) + { + break; + } + offset += size; + } + return getCompilationUnit(offset); +} + + Dwarf::DIEAbbreviation Dwarf::getAbbreviation(uint64_t code, uint64_t offset) const { // Linear search in the .debug_abbrev section, starting at offset @@ -590,7 +862,7 @@ Dwarf::DIEAbbreviation Dwarf::getAbbreviation(uint64_t code, uint64_t offset) co SAFE_CHECK(false, "could not find abbreviation code"); } -Dwarf::AttributeValue Dwarf::readAttributeValue(std::string_view & sp, uint64_t form, bool is64Bit) const +Dwarf::AttributeValue Dwarf::readAttributeValue(std::string_view & sp, uint64_t form, bool is64_bit) const { switch (form) { @@ -628,26 +900,18 @@ Dwarf::AttributeValue Dwarf::readAttributeValue(std::string_view & sp, uint64_t return uint64_t(1); case DW_FORM_sec_offset: [[fallthrough]]; case DW_FORM_ref_addr: - return readOffset(sp, is64Bit); + return readOffset(sp, is64_bit); case DW_FORM_string: return readNullTerminated(sp); case DW_FORM_strp: - return getStringFromStringSection(readOffset(sp, is64Bit)); + return getStringFromStringSection(str_, readOffset(sp, is64_bit)); case DW_FORM_indirect: // form is explicitly specified - return readAttributeValue(sp, readULEB(sp), is64Bit); + return readAttributeValue(sp, readULEB(sp), is64_bit); default: SAFE_CHECK(false, "invalid attribute form"); } } -std::string_view Dwarf::getStringFromStringSection(uint64_t offset) const -{ - SAFE_CHECK(offset < strings_.size(), "invalid strp offset"); - std::string_view sp(strings_); - sp.remove_prefix(offset); - return readNullTerminated(sp); -} - /** * Find @address in .debug_aranges and return the offset in * .debug_info for compilation unit to which this address belongs. @@ -724,7 +988,7 @@ bool Dwarf::findLocation( // Partial compilation unit (DW_TAG_partial_unit) is not supported. SAFE_CHECK(die.abbr.tag == DW_TAG_compile_unit, "expecting compile unit entry"); - // Read attributes, extracting the few we care about + // Offset in .debug_line for the line number VM program for this CU std::optional line_offset = 0; std::string_view compilation_directory; std::optional main_file_name; @@ -772,7 +1036,7 @@ bool Dwarf::findLocation( std::string_view line_section(line_); line_section.remove_prefix(*line_offset); - LineNumberVM line_vm(line_section, compilation_directory); + LineNumberVM line_vm(line_section, compilation_directory, str_, line_str_); // Execute line number VM program to find file and line info.has_file_and_line = line_vm.findAddress(address, info.file, info.line); @@ -863,8 +1127,11 @@ bool Dwarf::findLocation( return info.has_file_and_line; } -void Dwarf::findSubProgramDieForAddress( - const CompilationUnit & cu, const Die & die, uint64_t address, std::optional base_addr_cu, Die & subprogram) const +void Dwarf::findSubProgramDieForAddress(const CompilationUnit & cu, + const Die & die, + uint64_t address, + std::optional base_addr_cu, + Die & subprogram) const { forEachChild(cu, die, [&](const Die & child_die) { @@ -885,9 +1152,14 @@ void Dwarf::findSubProgramDieForAddress( low_pc = std::get(attr.attr_value); break; case DW_AT_high_pc: - // Value of DW_AT_high_pc attribute can be an address - // (DW_FORM_addr) or an offset (DW_FORM_data). - is_high_pc_addr = (attr.spec.form == DW_FORM_addr); + // The value of the DW_AT_high_pc attribute can be + // an address (DW_FORM_addr*) or an offset (DW_FORM_data*). + is_high_pc_addr = attr.spec.form == DW_FORM_addr || // + attr.spec.form == DW_FORM_addrx || // + attr.spec.form == DW_FORM_addrx1 || // + attr.spec.form == DW_FORM_addrx2 || // + attr.spec.form == DW_FORM_addrx3 || // + attr.spec.form == DW_FORM_addrx4; high_pc = std::get(attr.attr_value); break; } @@ -896,7 +1168,7 @@ void Dwarf::findSubProgramDieForAddress( }); bool pc_match = low_pc && high_pc && is_high_pc_addr && address >= *low_pc && (address < (*is_high_pc_addr ? *high_pc : *low_pc + *high_pc)); - bool range_match = range_offset && isAddrInRangeList(address, base_addr_cu, range_offset.value(), cu.addr_size); + bool range_match = range_offset && isAddrInRangeList(cu, address, base_addr_cu, range_offset.value(), cu.addr_size); if (pc_match || range_match) { subprogram = child_die; @@ -971,9 +1243,14 @@ void Dwarf::findInlinedSubroutineDieForAddress( low_pc = std::get(attr.attr_value); break; case DW_AT_high_pc: - // Value of DW_AT_high_pc attribute can be an address - // (DW_FORM_addr) or an offset (DW_FORM_data). - is_high_pc_addr = (attr.spec.form == DW_FORM_addr); + // The value of the DW_AT_high_pc attribute can be + // an address (DW_FORM_addr*) or an offset (DW_FORM_data*). + is_high_pc_addr = attr.spec.form == DW_FORM_addr || // + attr.spec.form == DW_FORM_addrx || // + attr.spec.form == DW_FORM_addrx1 || // + attr.spec.form == DW_FORM_addrx2 || // + attr.spec.form == DW_FORM_addrx3 || // + attr.spec.form == DW_FORM_addrx4; high_pc = std::get(attr.attr_value); break; case DW_AT_abstract_origin: @@ -1005,7 +1282,7 @@ void Dwarf::findInlinedSubroutineDieForAddress( // TODO: Support relocated address which requires lookup in relocation map. bool pc_match = low_pc && high_pc && is_high_pc_addr && address >= *low_pc && (address < (*is_high_pc_addr ? *high_pc : *low_pc + *high_pc)); - bool range_match = range_offset && isAddrInRangeList(address, base_addr_cu, range_offset.value(), cu.addr_size); + bool range_match = range_offset && isAddrInRangeList(cu, address, base_addr_cu, range_offset.value(), cu.addr_size); if (!pc_match && !range_match) { // Address doesn't match. Keep searching other children. @@ -1107,7 +1384,7 @@ void Dwarf::findInlinedSubroutineDieForAddress( // Not applicable for DW_AT_abstract_origin. location.name = (*abstract_origin_ref_type != DW_FORM_ref_addr) ? get_function_name(cu, cu.offset + *abstract_origin) - : get_function_name(findCompilationUnit(info_, *abstract_origin), *abstract_origin); + : get_function_name(findCompilationUnit(*abstract_origin), *abstract_origin); /// FIXME: see comment above if (die_for_inline_broken) @@ -1144,7 +1421,11 @@ bool Dwarf::findAddress( if (findDebugInfoOffset(address, aranges_, offset)) { // Read compilation unit header from .debug_info - auto unit = getCompilationUnit(info_, offset); + auto unit = getCompilationUnit(offset); + if (unit.unit_type != DW_UT_compile && unit.unit_type != DW_UT_skeleton) + { + return false; + } findLocation(address, mode, unit, locationInfo, inline_frames); return locationInfo.has_file_and_line; } @@ -1168,84 +1449,160 @@ bool Dwarf::findAddress( uint64_t offset = 0; while (offset < info_.size() && !locationInfo.has_file_and_line) { - auto unit = getCompilationUnit(info_, offset); + auto unit = getCompilationUnit(offset); offset += unit.size; + if (unit.unit_type != DW_UT_compile && unit.unit_type != DW_UT_skeleton) + { + continue; + } findLocation(address, mode, unit, locationInfo, inline_frames); } return locationInfo.has_file_and_line; } -bool Dwarf::isAddrInRangeList(uint64_t address, std::optional base_addr, size_t offset, uint8_t addr_size) const +bool Dwarf::isAddrInRangeList(const CompilationUnit & cu, + uint64_t address, + std::optional base_addr, + size_t offset, + uint8_t addr_size) const { SAFE_CHECK(addr_size == 4 || addr_size == 8, "wrong address size"); - if (ranges_.empty()) + if (cu.version <= 4 && !ranges_.empty()) { - return false; - } + const bool is64_bit_addr = addr_size == 8; + std::string_view sp = ranges_; + sp.remove_prefix(offset); + const uint64_t max_addr = is64_bit_addr ? std::numeric_limits::max() : std::numeric_limits::max(); + while (!sp.empty()) + { + uint64_t begin = readOffset(sp, is64_bit_addr); + uint64_t end = readOffset(sp, is64_bit_addr); + // The range list entry is a base address selection entry. + if (begin == max_addr) + { + base_addr = end; + continue; + } + // The range list entry is an end of list entry. + if (begin == 0 && end == 0) + { + break; + } - const bool is_64bit_addr = addr_size == 8; - std::string_view sp = ranges_; - sp.remove_prefix(offset); - const uint64_t max_addr = is_64bit_addr ? std::numeric_limits::max() : std::numeric_limits::max(); - while (!sp.empty()) - { - uint64_t begin = readOffset(sp, is_64bit_addr); - uint64_t end = readOffset(sp, is_64bit_addr); - // The range list entry is a base address selection entry. - if (begin == max_addr) - { - base_addr = end; - continue; - } - // The range list entry is an end of list entry. - if (begin == 0 && end == 0) - { - break; - } - // Check if the given address falls in the range list entry. - // 2.17.3 Non-Contiguous Address Ranges - // The applicable base address of a range list entry is determined by the - // closest preceding base address selection entry (see below) in the same - // range list. If there is no such selection entry, then the applicable base - // address defaults to the base address of the compilation unit. - if (base_addr && address >= begin + *base_addr && address < end + *base_addr) - { - return true; + // Check if the given address falls in the range list entry. + // 2.17.3 Non-Contiguous Address Ranges + // The applicable base address of a range list entry is determined by the + // closest preceding base address selection entry (see below) in the same + // range list. If there is no such selection entry, then the applicable + // base address defaults to the base address of the compilation unit. + if (base_addr && address >= begin + *base_addr && address < end + *base_addr) + { + return true; + } } } + if (cu.version == 5 && !rnglists_.empty() && cu.addr_base.has_value()) + { + auto rnglists = rnglists_; + rnglists.remove_prefix(offset); + + while (!rnglists.empty()) + { + auto kind = read(rnglists); + switch (kind) + { + case DW_RLE_end_of_list: + return false; + case DW_RLE_base_addressx: { + auto index = readULEB(rnglists); + auto sp = addr_.substr(*cu.addr_base + index * sizeof(uint64_t)); + base_addr = read(sp); + } + break; + + case DW_RLE_startx_endx: { + auto index_start = readULEB(rnglists); + auto index_end = readULEB(rnglists); + auto sp_start = addr_.substr(*cu.addr_base + index_start * sizeof(uint64_t)); + auto start = read(sp_start); + + auto sp_end = addr_.substr(*cu.addr_base + index_end * sizeof(uint64_t)); + auto end = read(sp_end); + if (address >= start && address < end) + { + return true; + } + } + break; + + case DW_RLE_startx_length: { + auto index_start = readULEB(rnglists); + auto length = readULEB(rnglists); + auto sp_start = addr_.substr(*cu.addr_base + index_start * sizeof(uint64_t)); + auto start = read(sp_start); + + auto sp_end = addr_.substr(*cu.addr_base + index_start * sizeof(uint64_t) + length); + auto end = read(sp_end); + if (start != end && address >= start && address < end) + { + return true; + } + } + break; + + case DW_RLE_offset_pair: { + auto offset_start = readULEB(rnglists); + auto offset_end = readULEB(rnglists); + if (base_addr && address >= (*base_addr + offset_start) && address < (*base_addr + offset_end)) + { + return true; + } + } + break; + + case DW_RLE_base_address: + base_addr = read(rnglists); + break; + + case DW_RLE_start_end: { + uint64_t start = read(rnglists); + uint64_t end = read(rnglists); + if (address >= start && address < end) + { + return true; + } + } + break; + + case DW_RLE_start_length: { + uint64_t start = read(rnglists); + uint64_t end = start + readULEB(rnglists); + if (address >= start && address < end) + { + return true; + } + } + break; + + default: + SAFE_CHECK(false, "Unexpected debug_rnglists entry kind"); + } + } + } return false; } -// static -Dwarf::CompilationUnit Dwarf::findCompilationUnit(std::string_view info, uint64_t targetOffset) -{ - SAFE_CHECK(targetOffset < info.size(), "unexpected target address"); - uint64_t offset = 0; - while (offset < info.size()) - { - std::string_view chunk(info); - chunk.remove_prefix(offset); - auto initial_length = read(chunk); - auto is_64bit = (initial_length == uint32_t(-1)); - auto size = is_64bit ? read(chunk) : initial_length; - SAFE_CHECK(size <= chunk.size(), "invalid chunk size"); - size += is_64bit ? 12 : 4; - - if (offset + size > targetOffset) - { - break; - } - offset += size; - } - return getCompilationUnit(info, offset); -} - - -Dwarf::LineNumberVM::LineNumberVM(std::string_view data, std::string_view compilationDirectory) +Dwarf::LineNumberVM::LineNumberVM( + std::string_view data, + std::string_view compilationDirectory, + std::string_view debugStr, + std::string_view debugLineStr) : compilationDirectory_(compilationDirectory) + , debugStr_(debugStr) + , debugLineStr_(debugLineStr) { Section section(data); SAFE_CHECK(section.next(data_), "invalid line number VM"); @@ -1269,17 +1626,154 @@ void Dwarf::LineNumberVM::reset() discriminator_ = 0; } +struct LineNumberAttribute +{ + uint64_t content_type_code; + uint64_t form_code; + std::variant attr_value; +}; + +LineNumberAttribute readLineNumberAttribute( + bool is64_bit, std::string_view & format, std::string_view & entries, std::string_view debugStr, std::string_view debugLineStr) +{ + uint64_t content_type_code = readULEB(format); + uint64_t form_code = readULEB(format); + std::variant attr_value; + + switch (content_type_code) + { + case DW_LNCT_path: { + switch (form_code) + { + case DW_FORM_string: + attr_value = readNullTerminated(entries); + break; + case DW_FORM_line_strp: { + auto off = readOffset(entries, is64_bit); + attr_value = getStringFromStringSection(debugLineStr, off); + } + break; + case DW_FORM_strp: + attr_value = getStringFromStringSection(debugStr, readOffset(entries, is64_bit)); + break; + case DW_FORM_strp_sup: + SAFE_CHECK(false, "Unexpected DW_FORM_strp_sup"); + break; + default: + SAFE_CHECK(false, "Unexpected form for DW_LNCT_path"); + break; + } + } + break; + + case DW_LNCT_directory_index: { + switch (form_code) + { + case DW_FORM_data1: + attr_value = read(entries); + break; + case DW_FORM_data2: + attr_value = read(entries); + break; + case DW_FORM_udata: + attr_value = readULEB(entries); + break; + default: + SAFE_CHECK(false, "Unexpected form for DW_LNCT_directory_index"); + break; + } + } + break; + + case DW_LNCT_timestamp: { + switch (form_code) + { + case DW_FORM_udata: + attr_value = readULEB(entries); + break; + case DW_FORM_data4: + attr_value = read(entries); + break; + case DW_FORM_data8: + attr_value = read(entries); + break; + case DW_FORM_block: + attr_value = readBytes(entries, readULEB(entries)); + break; + default: + SAFE_CHECK(false, "Unexpected form for DW_LNCT_timestamp"); + } + } + break; + + case DW_LNCT_size: { + switch (form_code) + { + case DW_FORM_udata: + attr_value = readULEB(entries); + break; + case DW_FORM_data1: + attr_value = read(entries); + break; + case DW_FORM_data2: + attr_value = read(entries); + break; + case DW_FORM_data4: + attr_value = read(entries); + break; + case DW_FORM_data8: + attr_value = read(entries); + break; + default: + SAFE_CHECK(false, "Unexpected form for DW_LNCT_size"); + break; + } + } + break; + + case DW_LNCT_MD5: { + switch (form_code) + { + case DW_FORM_data16: + attr_value = readBytes(entries, 16); + break; + default: + SAFE_CHECK(false, "Unexpected form for DW_LNCT_MD5"); + break; + } + } + break; + + default: + // TODO: skip over vendor data as specified by the form instead. + SAFE_CHECK(false, "Unexpected vendor content type code"); + break; + } + return { + .content_type_code = content_type_code, + .form_code = form_code, + .attr_value = attr_value, + }; +} + void Dwarf::LineNumberVM::init() { version_ = read(data_); - SAFE_CHECK(version_ >= 2 && version_ <= 4, "invalid version in line number VM"); + SAFE_CHECK(version_ >= 2 && version_ <= 5, "invalid version in line number VM: {}", version_); + if (version_ == 5) + { + auto address_size = read(data_); + SAFE_CHECK(address_size == sizeof(uintptr_t), "Unexpected Line Number Table address_size"); + auto segment_selector_size = read(data_); + SAFE_CHECK(segment_selector_size == 0, "Segments not supported"); + } uint64_t header_length = readOffset(data_, is64Bit_); SAFE_CHECK(header_length <= data_.size(), "invalid line number VM header length"); std::string_view header(data_.data(), header_length); data_ = std::string_view(header.end(), data_.end() - header.end()); minLength_ = read(header); - if (version_ == 4) + if (version_ >= 4) { // Version 2 and 3 records don't have this uint8_t max_ops_per_instruction = read(header); SAFE_CHECK(max_ops_per_instruction == 1, "VLIW not supported"); @@ -1292,26 +1786,75 @@ void Dwarf::LineNumberVM::init() standardOpcodeLengths_ = reinterpret_cast(header.data()); //-V506 header.remove_prefix(opcodeBase_ - 1); - // We don't want to use heap, so we don't keep an unbounded amount of state. - // We'll just skip over include directories and file names here, and - // we'll loop again when we actually need to retrieve one. - std::string_view sp; - const char * tmp = header.data(); - includeDirectoryCount_ = 0; - while (!(sp = readNullTerminated(header)).empty()) + if (version_ <= 4) { - ++includeDirectoryCount_; - } - includeDirectories_ = std::string_view(tmp, header.data() - tmp); + // We don't want to use heap, so we don't keep an unbounded amount of state. + // We'll just skip over include directories and file names here, and + // we'll loop again when we actually need to retrieve one. + std::string_view sp; + const char * tmp = header.data(); + v4_.includeDirectoryCount = 0; + while (!(sp = readNullTerminated(header)).empty()) + { + ++v4_.includeDirectoryCount; + } + v4_.includeDirectories = {tmp, header.data()}; - tmp = header.data(); - FileName fn; - fileNameCount_ = 0; - while (readFileName(header, fn)) - { - ++fileNameCount_; + tmp = header.data(); + FileName fn; + v4_.fileNameCount = 0; + while (readFileName(header, fn)) + { + ++v4_.fileNameCount; + } + v4_.fileNames = {tmp, header.data()}; + } + else if (version_ == 5) + { + v5_.directoryEntryFormatCount = read(header); + const char * tmp = header.data(); + for (uint8_t i = 0; i < v5_.directoryEntryFormatCount; i++) + { + // A sequence of directory entry format descriptions. Each description + // consists of a pair of ULEB128 values: + readULEB(header); // A content type code + readULEB(header); // A form code using the attribute form codes + } + v5_.directoryEntryFormat = {tmp, header.data()}; + v5_.directoriesCount = readULEB(header); + tmp = header.data(); + for (uint64_t i = 0; i < v5_.directoriesCount; i++) + { + std::string_view format = v5_.directoryEntryFormat; + for (uint8_t f = 0; f < v5_.directoryEntryFormatCount; f++) + { + readLineNumberAttribute(is64Bit_, format, header, debugStr_, debugLineStr_); + } + } + v5_.directories = {tmp, header.data()}; + + v5_.fileNameEntryFormatCount = read(header); + tmp = header.data(); + for (uint8_t i = 0; i < v5_.fileNameEntryFormatCount; i++) + { + // A sequence of file entry format descriptions. Each description + // consists of a pair of ULEB128 values: + readULEB(header); // A content type code + readULEB(header); // A form code using the attribute form codes + } + v5_.fileNameEntryFormat = {tmp, header.data()}; + v5_.fileNamesCount = readULEB(header); + tmp = header.data(); + for (uint64_t i = 0; i < v5_.fileNamesCount; i++) + { + std::string_view format = v5_.fileNameEntryFormat; + for (uint8_t f = 0; f < v5_.fileNameEntryFormatCount; f++) + { + readLineNumberAttribute(is64Bit_, format, header, debugStr_, debugLineStr_); + } + } + v5_.fileNames = {tmp, header.data()}; } - fileNames_ = std::string_view(tmp, header.data() - tmp); } bool Dwarf::LineNumberVM::next(std::string_view & program) @@ -1327,54 +1870,110 @@ bool Dwarf::LineNumberVM::next(std::string_view & program) Dwarf::LineNumberVM::FileName Dwarf::LineNumberVM::getFileName(uint64_t index) const { - SAFE_CHECK(index != 0, "invalid file index 0"); - - FileName fn; - if (index <= fileNameCount_) + if (version_ <= 4) { - std::string_view file_names = fileNames_; + SAFE_CHECK(index != 0, "invalid file index 0"); + FileName fn; + if (index <= v4_.fileNameCount) + { + std::string_view file_names = v4_.fileNames; + for (; index; --index) + { + if (!readFileName(file_names, fn)) + { + abort(); + } + } + return fn; + } + + index -= v4_.fileNameCount; + + std::string_view program = data_; for (; index; --index) { - if (!readFileName(file_names, fn)) + SAFE_CHECK(nextDefineFile(program, fn), "invalid file index"); + } + + return fn; + } + else + { + FileName fn; + SAFE_CHECK(index < v5_.fileNamesCount, "invalid file index"); + std::string_view file_names = v5_.fileNames; + for (uint64_t i = 0; i < v5_.fileNamesCount; i++) + { + std::string_view format = v5_.fileNameEntryFormat; + for (uint8_t f = 0; f < v5_.fileNameEntryFormatCount; f++) { - abort(); + auto attr = readLineNumberAttribute(is64Bit_, format, file_names, debugStr_, debugLineStr_); + if (i == index) + { + switch (attr.content_type_code) + { + case DW_LNCT_path: + fn.relativeName = std::get(attr.attr_value); + break; + case DW_LNCT_directory_index: + fn.directoryIndex = std::get(attr.attr_value); + break; + } + } } } return fn; } - - index -= fileNameCount_; - - std::string_view program = data_; - for (; index; --index) - { - SAFE_CHECK(nextDefineFile(program, fn), "invalid file index"); - } - - return fn; } std::string_view Dwarf::LineNumberVM::getIncludeDirectory(uint64_t index) const { - if (index == 0) + if (version_ <= 4) { - return std::string_view(); - } - - SAFE_CHECK(index <= includeDirectoryCount_, "invalid include directory"); - - std::string_view include_directories = includeDirectories_; - std::string_view dir; - for (; index; --index) - { - dir = readNullTerminated(include_directories); - if (dir.empty()) + if (index == 0) { - abort(); // BUG + // In DWARF <= 4 the current directory is not represented in the + // directories field and a directory index of 0 implicitly referred to + // that directory as found in the DW_AT_comp_dir attribute of the + // compilation unit debugging information entry. + return {}; } - } - return dir; + SAFE_CHECK(index <= v4_.includeDirectoryCount, "invalid include directory"); + + std::string_view include_directories = v4_.includeDirectories; + std::string_view dir; + for (; index; --index) + { + dir = readNullTerminated(include_directories); + if (dir.empty()) + { + abort(); // BUG + } + } + + return dir; + } + else + { + SAFE_CHECK(index < v5_.directoriesCount, "invalid file index"); + std::string_view directories = v5_.directories; + for (uint64_t i = 0; i < v5_.directoriesCount; i++) + { + std::string_view format = v5_.directoryEntryFormat; + for (uint8_t f = 0; f < v5_.directoryEntryFormatCount; f++) + { + auto attr = readLineNumberAttribute(is64Bit_, format, directories, debugStr_, debugLineStr_); + if (i == index && attr.content_type_code == DW_LNCT_path) + { + return std::get(attr.attr_value); + } + } + } + // This could only happen if DWARF5's directory_entry_format doesn't contain + // a DW_LNCT_path. Highly unlikely, but we shouldn't crash. + return std::string_view(""); + } } bool Dwarf::LineNumberVM::readFileName(std::string_view & program, FileName & fn) @@ -1422,6 +2021,7 @@ bool Dwarf::LineNumberVM::nextDefineFile(std::string_view & program, FileName & if (opcode == DW_LNE_define_file) { + SAFE_CHECK(version_ < 5, "DW_LNE_define_file deprecated in DWARF5"); SAFE_CHECK(readFileName(program, fn), "invalid empty file in DW_LNE_define_file"); return true; } @@ -1535,6 +2135,7 @@ Dwarf::LineNumberVM::StepResult Dwarf::LineNumberVM::step(std::string_view & pro address_ = read(program); return CONTINUE; case DW_LNE_define_file: + SAFE_CHECK(version_ < 5, "DW_LNE_define_file deprecated in DWARF5"); // We can't process DW_LNE_define_file here, as it would require us to // use unbounded amounts of state (ie. use the heap). We'll do a second // pass (using nextDefineFile()) if necessary. @@ -1549,6 +2150,16 @@ Dwarf::LineNumberVM::StepResult Dwarf::LineNumberVM::step(std::string_view & pro return CONTINUE; } +Dwarf::Path Dwarf::LineNumberVM::getFullFileName(uint64_t index) const +{ + auto fn = getFileName(index); + // DWARF <= 4: the current dir is not represented in the CU's Line Number + // Program Header and relies on the CU's DW_AT_comp_dir. + // DWARF 5: the current directory is explicitly present. + const std::string_view base_dir = version_ == 5 ? "" : compilationDirectory_; + return Path(base_dir, getIncludeDirectory(fn.directoryIndex), fn.relativeName); +} + bool Dwarf::LineNumberVM::findAddress(uintptr_t target, Path & file, uint64_t & line) { std::string_view program = data_; @@ -1588,12 +2199,18 @@ bool Dwarf::LineNumberVM::findAddress(uintptr_t target, Path & file, uint64_t & // Found it! Note that ">" is indeed correct (not ">="), as each // sequence is guaranteed to have one entry past-the-end (emitted by // DW_LNE_end_sequence) - if (prev_file == 0) + // + // NOTE: In DWARF <= 4 the file register is non-zero. + // See DWARF 4: 6.2.4 The Line Number Program Header + // "The line number program assigns numbers to each of the file + // entries in order, beginning with 1, and uses those numbers instead + // of file names in the file register." + // DWARF 5 has a different include directory/file header and 0 is valid. + if (version_ <= 4 && prev_file == 0) { return false; } - auto fn = getFileName(prev_file); - file = Path(compilationDirectory_, getIncludeDirectory(fn.directoryIndex), fn.relativeName); + file = getFullFileName(prev_file); line = prev_line; return true; } diff --git a/src/Common/Dwarf.h b/src/Common/Dwarf.h index 6e3d3e74e81..09178c66d47 100644 --- a/src/Common/Dwarf.h +++ b/src/Common/Dwarf.h @@ -19,6 +19,7 @@ */ /** This file was edited for ClickHouse. + * Original is from folly library. */ #include @@ -113,8 +114,8 @@ public: // seems as the same path can be represented in multiple ways private: std::string_view baseDir_; /// NOLINT - std::string_view subDir_; /// NOLINT - std::string_view file_; /// NOLINT + std::string_view subDir_; /// NOLINT + std::string_view file_; /// NOLINT }; // Indicates inline function `name` is called at `line@file`. @@ -171,8 +172,6 @@ public: private: static bool findDebugInfoOffset(uintptr_t address, std::string_view aranges, uint64_t & offset); - void init(); - std::shared_ptr elf_; /// NOLINT // DWARF section made up of chunks, each prefixed with a length header. @@ -228,6 +227,7 @@ private: { uint64_t name = 0; uint64_t form = 0; + int64_t implicitConst = 0; // only set when form=DW_FORM_implicit_const explicit operator bool() const { return name != 0 || form != 0; } }; @@ -239,25 +239,43 @@ private: std::variant attr_value; }; + enum + { + DW_UT_compile = 0x01, + DW_UT_skeleton = 0x04, + }; + struct CompilationUnit { - bool is64Bit; /// NOLINT - uint8_t version; - uint8_t addr_size; + bool is64Bit = false; /// NOLINT + uint8_t version = 0; + uint8_t unit_type = DW_UT_compile; // DW_UT_compile or DW_UT_skeleton + uint8_t addr_size = 0; // Offset in .debug_info of this compilation unit. - uint32_t offset; - uint32_t size; + uint32_t offset = 0; + uint32_t size = 0; // Offset in .debug_info for the first DIE in this compilation unit. - uint32_t first_die; - uint64_t abbrev_offset; + uint32_t first_die = 0; + uint64_t abbrev_offset = 0; + + // The beginning of the CU's contribution to .debug_addr + std::optional addr_base; // DW_AT_addr_base (DWARF 5) + // The beginning of the offsets table (immediately following the + // header) of the CU's contribution to .debug_loclists + std::optional loclists_base; // DW_AT_loclists_base (DWARF 5) + // The beginning of the offsets table (immediately following the + // header) of the CU's contribution to .debug_rnglists + std::optional rnglists_base; // DW_AT_rnglists_base (DWARF 5) + // Points to the first string offset of the compilation unit’s + // contribution to the .debug_str_offsets (or .debug_str_offsets.dwo) section. + std::optional str_offsets_base; // DW_AT_str_offsets_base (DWARF 5) + // Only the CompilationUnit that contains the caller functions needs this cache. // Indexed by (abbr.code - 1) if (abbr.code - 1) < abbrCache.size(); std::vector abbr_cache; }; - static CompilationUnit getCompilationUnit(std::string_view info, uint64_t offset); - - /** cu must exist during the life cycle of created detail::Die. */ + /** cu must exist during the life cycle of created Die. */ Die getDieAtOffset(const CompilationUnit & cu, uint64_t offset) const; bool findLocation( @@ -278,16 +296,16 @@ private: class LineNumberVM { public: - LineNumberVM(std::string_view data, std::string_view compilationDirectory); + LineNumberVM( + std::string_view data, + std::string_view compilationDirectory, + std::string_view debugStr, + std::string_view debugLineStr); bool findAddress(uintptr_t target, Path & file, uint64_t & line); /** Gets full file name at given index including directory. */ - Path getFullFileName(uint64_t index) const - { - auto fn = getFileName(index); - return Path({}, getIncludeDirectory(fn.directoryIndex), fn.relativeName); - } + Path getFullFileName(uint64_t index) const; private: void init(); @@ -327,24 +345,42 @@ private: bool nextDefineFile(std::string_view & program, FileName & fn) const; // Initialization - bool is64Bit_; /// NOLINT - std::string_view data_; /// NOLINT - std::string_view compilationDirectory_; /// NOLINT + bool is64Bit_; /// NOLINT + std::string_view data_; /// NOLINT + std::string_view compilationDirectory_; /// NOLINT + std::string_view debugStr_; // needed for DWARF 5 /// NOLINT + std::string_view debugLineStr_; // DWARF 5 /// NOLINT // Header - uint16_t version_; /// NOLINT - uint8_t minLength_; /// NOLINT + uint16_t version_; /// NOLINT + uint8_t minLength_; /// NOLINT bool defaultIsStmt_; /// NOLINT - int8_t lineBase_; /// NOLINT - uint8_t lineRange_; /// NOLINT + int8_t lineBase_; /// NOLINT + uint8_t lineRange_; /// NOLINT uint8_t opcodeBase_; /// NOLINT const uint8_t * standardOpcodeLengths_; /// NOLINT - std::string_view includeDirectories_; /// NOLINT - size_t includeDirectoryCount_; /// NOLINT + // 6.2.4 The Line Number Program Header. + struct + { + size_t includeDirectoryCount; + std::string_view includeDirectories; + size_t fileNameCount; + std::string_view fileNames; + } v4_; - std::string_view fileNames_; /// NOLINT - size_t fileNameCount_; /// NOLINT + struct + { + uint8_t directoryEntryFormatCount; + std::string_view directoryEntryFormat; + uint64_t directoriesCount; + std::string_view directories; + + uint8_t fileNameEntryFormatCount; + std::string_view fileNameEntryFormat; + uint64_t fileNamesCount; + std::string_view fileNames; + } v5_; // State machine registers uint64_t address_; /// NOLINT @@ -397,20 +433,26 @@ private: */ size_t forEachAttribute(const CompilationUnit & cu, const Die & die, std::function f) const; - Attribute readAttribute(const Die & die, AttributeSpec spec, std::string_view & info) const; + Attribute readAttribute( + const CompilationUnit & cu, + const Die & die, + AttributeSpec spec, + std::string_view & info) const; // Read one attribute pair, remove_prefix sp; returns <0, 0> at end. static AttributeSpec readAttributeSpec(std::string_view & sp); // Read one attribute value, remove_prefix sp using AttributeValue = std::variant; - AttributeValue readAttributeValue(std::string_view & sp, uint64_t form, bool is64Bit) const; + AttributeValue readAttributeValue(std::string_view & sp, uint64_t form, bool is64_bit) const; // Get an ELF section by name, return true if found - bool getSection(const char * name, std::string_view * section) const; + std::string_view getSection(const char * name) const; + + CompilationUnit getCompilationUnit(uint64_t offset) const; + // Finds the Compilation Unit starting at offset. + CompilationUnit findCompilationUnit(uint64_t targetOffset) const; - // Get a string from the .debug_str section - std::string_view getStringFromStringSection(uint64_t offset) const; template std::optional getAttribute(const CompilationUnit & cu, const Die & die, uint64_t attr_name) const @@ -429,17 +471,24 @@ private: } // Check if the given address is in the range list at the given offset in .debug_ranges. - bool isAddrInRangeList(uint64_t address, std::optional base_addr, size_t offset, uint8_t addr_size) const; + bool isAddrInRangeList( + const CompilationUnit & cu, + uint64_t address, + std::optional base_addr, + size_t offset, + uint8_t addr_size) const; - // Finds the Compilation Unit starting at offset. - static CompilationUnit findCompilationUnit(std::string_view info, uint64_t targetOffset); - - std::string_view info_; // .debug_info /// NOLINT - std::string_view abbrev_; // .debug_abbrev /// NOLINT - std::string_view aranges_; // .debug_aranges /// NOLINT - std::string_view line_; // .debug_line /// NOLINT - std::string_view strings_; // .debug_str /// NOLINT - std::string_view ranges_; // .debug_ranges /// NOLINT + std::string_view abbrev_; // .debug_abbrev /// NOLINT + std::string_view addr_; // .debug_addr (DWARF 5) /// NOLINT + std::string_view aranges_; // .debug_aranges /// NOLINT + std::string_view info_; // .debug_info /// NOLINT + std::string_view line_; // .debug_line /// NOLINT + std::string_view line_str_; // .debug_line_str (DWARF 5) /// NOLINT + std::string_view loclists_; // .debug_loclists (DWARF 5) /// NOLINT + std::string_view ranges_; // .debug_ranges /// NOLINT + std::string_view rnglists_; // .debug_rnglists (DWARF 5) /// NOLINT + std::string_view str_; // .debug_str /// NOLINT + std::string_view str_offsets_; // .debug_str_offsets (DWARF 5) /// NOLINT }; } diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 8f46cea25a0..e80ad5c141a 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -636,6 +636,7 @@ M(665, CANNOT_CONNECT_NATS) \ M(666, CANNOT_USE_CACHE) \ M(667, NOT_INITIALIZED) \ + M(668, INVALID_STATE) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 3645ac5594f..931f06fdb51 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -176,10 +176,10 @@ static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string void tryLogCurrentException(const char * log_name, const std::string & start_of_message) { - /// Under high memory pressure, any new allocation will definitelly lead - /// to MEMORY_LIMIT_EXCEEDED exception. + /// Under high memory pressure, new allocations throw a + /// MEMORY_LIMIT_EXCEEDED exception. /// - /// And in this case the exception will not be logged, so let's block the + /// In this case the exception will not be logged, so let's block the /// MemoryTracker until the exception will be logged. LockMemoryExceptionInThread lock_memory_tracker(VariableContext::Global); @@ -189,8 +189,8 @@ void tryLogCurrentException(const char * log_name, const std::string & start_of_ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message) { - /// Under high memory pressure, any new allocation will definitelly lead - /// to MEMORY_LIMIT_EXCEEDED exception. + /// Under high memory pressure, new allocations throw a + /// MEMORY_LIMIT_EXCEEDED exception. /// /// And in this case the exception will not be logged, so let's block the /// MemoryTracker until the exception will be logged. diff --git a/src/Common/ExternalModelInfo.h b/src/Common/ExternalModelInfo.h new file mode 100644 index 00000000000..378e4984af6 --- /dev/null +++ b/src/Common/ExternalModelInfo.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/// Details about external machine learning model, used by clickhouse-server and clickhouse-library-bridge +struct ExternalModelInfo +{ + String model_path; + String model_type; + std::chrono::system_clock::time_point loading_start_time; /// serialized as std::time_t + std::chrono::milliseconds loading_duration; /// serialized as UInt64 +}; + +using ExternalModelInfos = std::vector; + +} diff --git a/src/Common/FieldVisitorConvertToNumber.h b/src/Common/FieldVisitorConvertToNumber.h index 92da0f89844..466d312406e 100644 --- a/src/Common/FieldVisitorConvertToNumber.h +++ b/src/Common/FieldVisitorConvertToNumber.h @@ -94,21 +94,7 @@ public: T operator() (const DecimalField & x) const { if constexpr (std::is_floating_point_v) - return x.getValue(). template convertTo() / x.getScaleMultiplier(). template convertTo(); - else if constexpr (std::is_same_v) - { - if constexpr (sizeof(U) < 16) - { - return UInt128(0, (x.getValue() / x.getScaleMultiplier()).value); - } - else if constexpr (sizeof(U) == 16) - { - auto tmp = (x.getValue() / x.getScaleMultiplier()).value; - return UInt128(tmp >> 64, UInt64(tmp)); - } - else - throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED); - } + return x.getValue().template convertTo() / x.getScaleMultiplier().template convertTo(); else return (x.getValue() / x.getScaleMultiplier()). template convertTo(); } @@ -134,4 +120,3 @@ public: }; } - diff --git a/src/Common/FieldVisitorSum.cpp b/src/Common/FieldVisitorSum.cpp index bc996ae2298..2c404c33177 100644 --- a/src/Common/FieldVisitorSum.cpp +++ b/src/Common/FieldVisitorSum.cpp @@ -15,7 +15,7 @@ FieldVisitorSum::FieldVisitorSum(const Field & rhs_) : rhs(rhs_) {} bool FieldVisitorSum::operator() (Int64 & x) const { return this->operator()(reinterpret_cast(x)); } bool FieldVisitorSum::operator() (UInt64 & x) const { - x += rhs.reinterpret(); + x += applyVisitor(FieldVisitorConvertToNumber(), rhs); return x != 0; } diff --git a/src/Common/FieldVisitorSum.h b/src/Common/FieldVisitorSum.h index cd8777e7bfb..c28e2058b05 100644 --- a/src/Common/FieldVisitorSum.h +++ b/src/Common/FieldVisitorSum.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -41,7 +42,7 @@ public: requires is_big_int_v bool operator() (T & x) const { - x += rhs.reinterpret(); + x += applyVisitor(FieldVisitorConvertToNumber(), rhs); return x != T(0); } }; diff --git a/src/Common/MemoryStatisticsOS.cpp b/src/Common/MemoryStatisticsOS.cpp index 22f8446121f..f2d2ab5fea9 100644 --- a/src/Common/MemoryStatisticsOS.cpp +++ b/src/Common/MemoryStatisticsOS.cpp @@ -135,7 +135,7 @@ MemoryStatisticsOS::Data MemoryStatisticsOS::get() const struct kinfo_proc kp; size_t len = sizeof(struct kinfo_proc); - if (-1 == ::sysctl(mib, 4, &kp, &len, NULL, 0)) + if (-1 == ::sysctl(mib, 4, &kp, &len, nullptr, 0)) throwFromErrno("Cannot sysctl(kern.proc.pid." + std::to_string(self) + ")", ErrorCodes::SYSTEM_ERROR); if (sizeof(struct kinfo_proc) != len) diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index cebec219ff2..e8573bcd343 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -12,6 +12,16 @@ #include #include +#include "config_core.h" + +#if USE_JEMALLOC +# include + +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY(x) STRINGIFY_HELPER(x) + +#endif + #include #include #include @@ -84,6 +94,7 @@ static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; MemoryTracker total_memory_tracker(nullptr, VariableContext::Global); +std::atomic MemoryTracker::free_memory_in_allocator_arenas; MemoryTracker::MemoryTracker(VariableContext level_) : parent(&total_memory_tracker), level(level_) {} MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_) : parent(parent_), level(level_) {} @@ -128,6 +139,16 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT if (MemoryTrackerBlockerInThread::isBlocked(level)) { + if (level == VariableContext::Global) + { + /// For global memory tracker always update memory usage. + amount.fetch_add(size, std::memory_order_relaxed); + + auto metric_loaded = metric.load(std::memory_order_relaxed); + if (metric_loaded != CurrentMetrics::end()) + CurrentMetrics::add(metric_loaded, size); + } + /// Since the MemoryTrackerBlockerInThread should respect the level, we should go to the next parent. if (auto * loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->allocImpl(size, throw_if_memory_exceeded, @@ -148,24 +169,6 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT Int64 current_hard_limit = hard_limit.load(std::memory_order_relaxed); Int64 current_profiler_limit = profiler_limit.load(std::memory_order_relaxed); - /// Cap the limit to the total_memory_tracker, since it may include some drift - /// for user-level memory tracker. - /// - /// And since total_memory_tracker is reset to the process resident - /// memory peridically (in AsynchronousMetrics::update()), any limit can be - /// capped to it, to avoid possible drift. - if (unlikely(current_hard_limit - && will_be > current_hard_limit - && level == VariableContext::User)) - { - Int64 total_amount = total_memory_tracker.get(); - if (amount > total_amount) - { - set(total_amount); - will_be = size + total_amount; - } - } - bool memory_limit_exceeded_ignored = false; bool allocation_traced = false; @@ -211,8 +214,30 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT memory_limit_exceeded_ignored = true; } + Int64 limit_to_check = current_hard_limit; - if (unlikely(current_hard_limit && will_be > current_hard_limit)) +#if USE_JEMALLOC + if (level == VariableContext::Global) + { + /// Jemalloc arenas may keep some extra memory. + /// This memory was substucted from RSS to decrease memory drift. + /// In case memory is close to limit, try to pugre the arenas. + /// This is needed to avoid OOM, because some allocations are directly done with mmap. + Int64 current_free_memory_in_allocator_arenas = free_memory_in_allocator_arenas.load(std::memory_order_relaxed); + + if (current_free_memory_in_allocator_arenas > 0 && current_hard_limit && current_free_memory_in_allocator_arenas + will_be > current_hard_limit) + { + if (free_memory_in_allocator_arenas.exchange(-current_free_memory_in_allocator_arenas) > 0) + { + mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); + } + } + + limit_to_check += abs(current_free_memory_in_allocator_arenas); + } +#endif + + if (unlikely(current_hard_limit && will_be > limit_to_check)) { if (memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) { @@ -311,6 +336,15 @@ void MemoryTracker::free(Int64 size) { if (MemoryTrackerBlockerInThread::isBlocked(level)) { + if (level == VariableContext::Global) + { + /// For global memory tracker always update memory usage. + amount.fetch_sub(size, std::memory_order_relaxed); + auto metric_loaded = metric.load(std::memory_order_relaxed); + if (metric_loaded != CurrentMetrics::end()) + CurrentMetrics::sub(metric_loaded, size); + } + /// Since the MemoryTrackerBlockerInThread should respect the level, we should go to the next parent. if (auto * loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->free(size); @@ -325,7 +359,7 @@ void MemoryTracker::free(Int64 size) } Int64 accounted_size = size; - if (level == VariableContext::Thread) + if (level == VariableContext::Thread || level == VariableContext::Global) { /// Could become negative if memory allocated in this thread is freed in another one amount.fetch_sub(accounted_size, std::memory_order_relaxed); @@ -396,12 +430,18 @@ void MemoryTracker::reset() } -void MemoryTracker::set(Int64 to) +void MemoryTracker::setRSS(Int64 rss_, Int64 free_memory_in_allocator_arenas_) { - amount.store(to, std::memory_order_relaxed); + Int64 new_amount = rss_; // - free_memory_in_allocator_arenas_; + total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); + free_memory_in_allocator_arenas.store(free_memory_in_allocator_arenas_, std::memory_order_relaxed); + + auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); + if (metric_loaded != CurrentMetrics::end()) + CurrentMetrics::set(metric_loaded, new_amount); bool log_memory_usage = true; - updatePeak(to, log_memory_usage); + total_memory_tracker.updatePeak(rss_, log_memory_usage); } diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index d9dd55a3a50..2d898935dcf 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -56,6 +56,8 @@ private: std::atomic hard_limit {0}; std::atomic profiler_limit {0}; + static std::atomic free_memory_in_allocator_arenas; + Int64 profiler_step = 0; /// To test exception safety of calling code, memory tracker throws an exception on each memory allocation with specified probability. @@ -199,8 +201,10 @@ public: /// Reset the accumulated data. void reset(); - /// Reset current counter to a new value. - void set(Int64 to); + /// Reset current counter to an RSS value. + /// Jemalloc may have pre-allocated arenas, they are accounted in RSS. + /// We can free this arenas in case of exception to avoid OOM. + static void setRSS(Int64 rss_, Int64 free_memory_in_allocator_arenas_); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp index 7a1f94926d5..d5c2188ad01 100644 --- a/src/Common/OpenTelemetryTraceContext.cpp +++ b/src/Common/OpenTelemetryTraceContext.cpp @@ -130,16 +130,15 @@ void SpanHolder::finish() noexcept try { auto log = current_thread_trace_context.span_log.lock(); - if (!log) + + /// The log might be disabled, check it before use + if (log) { - // The log might be disabled. - return; + this->finish_time_us + = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + + log->add(OpenTelemetrySpanLogElement(*this)); } - - this->finish_time_us - = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - - log->add(OpenTelemetrySpanLogElement(*this)); } catch (...) { diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index d3232d833ee..0baefad39e2 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -115,7 +115,13 @@ protected: } /// Minimum amount of memory to allocate for num_elements, including padding. - static size_t minimum_memory_for_elements(size_t num_elements) { return byte_size(num_elements) + pad_right + pad_left; } /// NOLINT + static size_t minimum_memory_for_elements(size_t num_elements) + { + size_t amount; + if (__builtin_add_overflow(byte_size(num_elements), pad_left + pad_right, &amount)) + throw Exception("Amount of memory requested to allocate is more than allowed", ErrorCodes::CANNOT_ALLOCATE_MEMORY); + return amount; + } void alloc_for_num_elements(size_t num_elements) /// NOLINT { @@ -225,9 +231,7 @@ public: void clear() { c_end = c_start; } template -#if defined(__clang__) ALWAYS_INLINE /// Better performance in clang build, worse performance in gcc build. -#endif void reserve(size_t n, TAllocatorParams &&... allocator_params) { if (n > capacity()) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 519fd95a266..46bec669626 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -146,6 +146,9 @@ M(SelectedRows, "Number of rows SELECTed from all tables.") \ M(SelectedBytes, "Number of bytes (uncompressed; for columns as they stored in memory) SELECTed from all tables.") \ \ + M(WaitMarksLoadMicroseconds, "Time spent loading marks") \ + M(BackgroundLoadingMarksTasks, "Number of background tasks for loading marks") \ + \ M(Merge, "Number of launched background merges.") \ M(MergedRows, "Rows read for background merges. This is the number of rows before merge.") \ M(MergedUncompressedBytes, "Uncompressed bytes (for columns as they stored in memory) that was read for background merges. This is the number before merge.") \ @@ -298,6 +301,30 @@ The server successfully detected this situation and will download merged part fr M(DiskS3WriteRequestsThrottling, "Number of 429 and 503 errors in POST, DELETE, PUT and PATCH requests to DiskS3 storage.") \ M(DiskS3WriteRequestsRedirects, "Number of redirects in POST, DELETE, PUT and PATCH requests to DiskS3 storage.") \ \ + M(S3DeleteObjects, "Number of S3 API DeleteObject(s) calls.") \ + M(S3CopyObject, "Number of S3 API CopyObject calls.") \ + M(S3ListObjects, "Number of S3 API ListObjects calls.") \ + M(S3HeadObject, "Number of S3 API HeadObject calls.") \ + M(S3CreateMultipartUpload, "Number of S3 API CreateMultipartUpload calls.") \ + M(S3UploadPartCopy, "Number of S3 API UploadPartCopy calls.") \ + M(S3UploadPart, "Number of S3 API UploadPart calls.") \ + M(S3AbortMultipartUpload, "Number of S3 API AbortMultipartUpload calls.") \ + M(S3CompleteMultipartUpload, "Number of S3 API CompleteMultipartUpload calls.") \ + M(S3PutObject, "Number of S3 API PutObject calls.") \ + M(S3GetObject, "Number of S3 API GetObject calls.") \ + \ + M(DiskS3DeleteObjects, "Number of DiskS3 API DeleteObject(s) calls.") \ + M(DiskS3CopyObject, "Number of DiskS3 API CopyObject calls.") \ + M(DiskS3ListObjects, "Number of DiskS3 API ListObjects calls.") \ + M(DiskS3HeadObject, "Number of DiskS3 API HeadObject calls.") \ + M(DiskS3CreateMultipartUpload, "Number of DiskS3 API CreateMultipartUpload calls.") \ + M(DiskS3UploadPartCopy, "Number of DiskS3 API UploadPartCopy calls.") \ + M(DiskS3UploadPart, "Number of DiskS3 API UploadPart calls.") \ + M(DiskS3AbortMultipartUpload, "Number of DiskS3 API AbortMultipartUpload calls.") \ + M(DiskS3CompleteMultipartUpload, "Number of DiskS3 API CompleteMultipartUpload calls.") \ + M(DiskS3PutObject, "Number of DiskS3 API PutObject calls.") \ + M(DiskS3GetObject, "Number of DiskS3 API GetObject calls.") \ + \ M(ReadBufferFromS3Microseconds, "Time spend in reading from S3.") \ M(ReadBufferFromS3Bytes, "Bytes read from S3.") \ M(ReadBufferFromS3RequestsErrors, "Number of exceptions while reading from S3.") \ @@ -408,6 +435,7 @@ The server successfully detected this situation and will download merged part fr M(OverflowThrow, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'throw' and exception was thrown.") \ M(OverflowAny, "Number of times approximate GROUP BY was in effect: when aggregation was performed only on top of first 'max_rows_to_group_by' unique keys and other keys were ignored due to 'group_by_overflow_mode' = 'any'.") \ + namespace ProfileEvents { diff --git a/src/Common/SettingConstraintWritability.h b/src/Common/SettingConstraintWritability.h new file mode 100644 index 00000000000..4a179dad286 --- /dev/null +++ b/src/Common/SettingConstraintWritability.h @@ -0,0 +1,23 @@ +#pragma once + + +namespace DB +{ + +enum class SettingConstraintWritability +{ + // Default. Setting can be change within specified range only in `readonly=0` or `readonly=2` mode. + WRITABLE, + + // Setting cannot be changed at all. + // Either READONLY or CONST keyword in SQL syntax can be used ( or in config.xml) to enable this. + // NOTE: name `CONST` is chosen to avoid confusion with `readonly` setting. + CONST, + + // Setting can be changed within specified range, regardless of `readonly` setting value. + CHANGEABLE_IN_READONLY, + + MAX +}; + +} diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index fc5377b3783..76ada9e0d75 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -264,6 +264,18 @@ protected: } }; +/// Schedule jobs/tasks on global thread pool without implicit passing tracing context on current thread to underlying worker as parent tracing context. +/// +/// If you implement your own job/task scheduling upon global thread pool or schedules a long time running job in a infinite loop way, +/// you need to use class, or you need to use ThreadFromGlobalPool below. +/// +/// See the comments of ThreadPool below to know how it works. +using ThreadFromGlobalPoolNoTracingContextPropagation = ThreadFromGlobalPoolImpl; + +/// An alias of thread that execute jobs/tasks on global thread pool by implicit passing tracing context on current thread to underlying worker as parent tracing context. +/// If jobs/tasks are directly scheduled by using APIs of this class, you need to use this class or you need to use class above. +using ThreadFromGlobalPool = ThreadFromGlobalPoolImpl; + /// Recommended thread pool for the case when multiple thread pools are created and destroyed. /// /// The template parameter of ThreadFromGlobalPool is set to false to disable tracing context propagation to underlying worker. @@ -274,9 +286,6 @@ protected: /// which means the tracing context initialized at underlying worker level won't be delete for a very long time. /// This would cause wrong context for further jobs scheduled in ThreadPool. /// -/// To make sure the tracing context are correctly propagated, we explicitly disable context propagation(including initialization and de-initialization) at underlying worker level. +/// To make sure the tracing context is correctly propagated, we explicitly disable context propagation(including initialization and de-initialization) at underlying worker level. /// -using ThreadPool = ThreadPoolImpl>; - -/// An alias for user code to execute a job in the global thread pool -using ThreadFromGlobalPool = ThreadFromGlobalPoolImpl; +using ThreadPool = ThreadPoolImpl; diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 6fcd3b52f16..55b793c2a70 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -605,7 +605,7 @@ void ZooKeeper::removeChildren(const std::string & path) } -void ZooKeeper::removeChildrenRecursive(const std::string & path, const String & keep_child_node) +void ZooKeeper::removeChildrenRecursive(const std::string & path, RemoveException keep_child) { Strings children = getChildren(path); while (!children.empty()) @@ -613,16 +613,23 @@ void ZooKeeper::removeChildrenRecursive(const std::string & path, const String & Coordination::Requests ops; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - removeChildrenRecursive(fs::path(path) / children.back()); - if (likely(keep_child_node.empty() || keep_child_node != children.back())) + if (keep_child.path.empty() || keep_child.path != children.back()) [[likely]] + { + removeChildrenRecursive(fs::path(path) / children.back()); ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1)); + } + else if (keep_child.remove_subtree) + { + removeChildrenRecursive(fs::path(path) / children.back()); + } + children.pop_back(); } multi(ops); } } -bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probably_flat, const String & keep_child_node) +bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probably_flat, RemoveException keep_child) { Strings children; if (tryGetChildren(path, children) != Coordination::Error::ZOK) @@ -639,16 +646,20 @@ bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probab { String child_path = fs::path(path) / children.back(); - /// Will try to avoid recursive getChildren calls if child_path probably has no children. - /// It may be extremely slow when path contain a lot of leaf children. - if (!probably_flat) - tryRemoveChildrenRecursive(child_path); - - if (likely(keep_child_node.empty() || keep_child_node != children.back())) + if (keep_child.path.empty() || keep_child.path != children.back()) [[likely]] { + /// Will try to avoid recursive getChildren calls if child_path probably has no children. + /// It may be extremely slow when path contain a lot of leaf children. + if (!probably_flat) + tryRemoveChildrenRecursive(child_path); + batch.push_back(child_path); ops.emplace_back(zkutil::makeRemoveRequest(child_path, -1)); } + else if (keep_child.remove_subtree && !probably_flat) + { + tryRemoveChildrenRecursive(child_path); + } children.pop_back(); } diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 1c7ba7f1d9c..791ae48b3f0 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -58,6 +58,18 @@ struct ShuffleHost } }; +struct RemoveException +{ + explicit RemoveException(std::string_view path_ = "", bool remove_subtree_ = true) + : path(path_) + , remove_subtree(remove_subtree_) + {} + + std::string_view path; + // whether we should keep the child node and its subtree or just the child node + bool remove_subtree; +}; + using GetPriorityForLoadBalancing = DB::GetPriorityForLoadBalancing; /// ZooKeeper session. The interface is substantially different from the usual libzookeeper API. @@ -219,13 +231,13 @@ public: void tryRemoveRecursive(const std::string & path); /// Similar to removeRecursive(...) and tryRemoveRecursive(...), but does not remove path itself. - /// If keep_child_node is not empty, this method will not remove path/keep_child_node (but will remove its subtree). - /// It can be useful to keep some child node as a flag which indicates that path is currently removing. - void removeChildrenRecursive(const std::string & path, const String & keep_child_node = {}); + /// Node defined as RemoveException will not be deleted. + void removeChildrenRecursive(const std::string & path, RemoveException keep_child = RemoveException{}); /// If probably_flat is true, this method will optimistically try to remove children non-recursive /// and will fall back to recursive removal if it gets ZNOTEMPTY for some child. /// Returns true if no kind of fallback happened. - bool tryRemoveChildrenRecursive(const std::string & path, bool probably_flat = false, const String & keep_child_node = {}); + /// Node defined as RemoveException will not be deleted. + bool tryRemoveChildrenRecursive(const std::string & path, bool probably_flat = false, RemoveException keep_child= RemoveException{}); /// Remove all children nodes (non recursive). void removeChildren(const std::string & path); diff --git a/src/Common/examples/compact_array.cpp b/src/Common/examples/compact_array.cpp index af6257e1963..58c4ea3be1e 100644 --- a/src/Common/examples/compact_array.cpp +++ b/src/Common/examples/compact_array.cpp @@ -1,9 +1,3 @@ -/// Bug in GCC: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Warray-bounds" -#endif - #include #include #include @@ -262,7 +256,3 @@ int main() runTests(); return 0; } - -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif diff --git a/src/Common/examples/parallel_aggregation.cpp b/src/Common/examples/parallel_aggregation.cpp index 045a385671b..f54c4cee12c 100644 --- a/src/Common/examples/parallel_aggregation.cpp +++ b/src/Common/examples/parallel_aggregation.cpp @@ -69,11 +69,6 @@ static void aggregate1(Map & map, Source::const_iterator begin, Source::const_it ++map[*it]; } -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - static void aggregate12(Map & map, Source::const_iterator begin, Source::const_iterator end) { Map::LookupResult found = nullptr; @@ -122,10 +117,6 @@ static void aggregate22(MapTwoLevel & map, Source::const_iterator begin, Source: } } -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif - static void merge2(MapTwoLevel * maps, size_t num_threads, size_t bucket) { for (size_t i = 1; i < num_threads; ++i) diff --git a/src/Common/examples/parallel_aggregation2.cpp b/src/Common/examples/parallel_aggregation2.cpp index 496331e203d..6c20f46ab0e 100644 --- a/src/Common/examples/parallel_aggregation2.cpp +++ b/src/Common/examples/parallel_aggregation2.cpp @@ -62,11 +62,6 @@ struct AggregateIndependent } }; -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - template struct AggregateIndependentWithSequentialKeysOptimization { @@ -115,11 +110,6 @@ struct AggregateIndependentWithSequentialKeysOptimization } }; -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif - - template struct MergeSequential { @@ -265,20 +255,11 @@ struct Creator void operator()(Value &) const {} }; -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - struct Updater { void operator()(Value & x) const { ++x; } }; -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif - struct Merger { void operator()(Value & dst, const Value & src) const { dst += src; } diff --git a/src/Coordination/CoordinationSettings.h b/src/Coordination/CoordinationSettings.h index 5247f5d7ec8..c436c1b6635 100644 --- a/src/Coordination/CoordinationSettings.h +++ b/src/Coordination/CoordinationSettings.h @@ -30,6 +30,7 @@ struct Settings; M(UInt64, snapshot_distance, 100000, "How many log items we have to collect to write new snapshot", 0) \ M(Bool, auto_forwarding, true, "Allow to forward write requests from followers to leader", 0) \ M(Milliseconds, shutdown_timeout, 5000, "How much time we will wait until RAFT shutdown", 0) \ + M(Milliseconds, session_shutdown_timeout, 10000, "How much time we will wait until sessions are closed during shutdown", 0) \ M(Milliseconds, startup_timeout, 180000, "How much time we will wait until RAFT to start.", 0) \ M(LogsLevel, raft_logs_level, LogsLevel::information, "Log internal RAFT logs into main server log level. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \ M(UInt64, rotate_log_storage_interval, 100000, "How many records will be stored in one log storage file", 0) \ diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 5b376a03b02..d725ecb5cfe 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -354,9 +354,6 @@ void KeeperDispatcher::shutdown() update_configuration_thread.join(); } - if (server) - server->shutdown(); - KeeperStorage::RequestForSession request_for_session; /// Set session expired for all pending requests @@ -368,10 +365,58 @@ void KeeperDispatcher::shutdown() setResponse(request_for_session.session_id, response); } - /// Clear all registered sessions - std::lock_guard lock(session_to_response_callback_mutex); - session_to_response_callback.clear(); + KeeperStorage::RequestsForSessions close_requests; + { + /// Clear all registered sessions + std::lock_guard lock(session_to_response_callback_mutex); + + if (server && hasLeader()) + { + close_requests.reserve(session_to_response_callback.size()); + // send to leader CLOSE requests for active sessions + for (const auto & [session, response] : session_to_response_callback) + { + auto request = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Close); + request->xid = Coordination::CLOSE_XID; + using namespace std::chrono; + KeeperStorage::RequestForSession request_info + { + .session_id = session, + .time = duration_cast(system_clock::now().time_since_epoch()).count(), + .request = std::move(request), + }; + + close_requests.push_back(std::move(request_info)); + } + } + + session_to_response_callback.clear(); + } + + // if there is no leader, there is no reason to do CLOSE because it's a write request + if (server && hasLeader() && !close_requests.empty()) + { + LOG_INFO(log, "Trying to close {} session(s)", close_requests.size()); + const auto raft_result = server->putRequestBatch(close_requests); + auto sessions_closing_done_promise = std::make_shared>(); + auto sessions_closing_done = sessions_closing_done_promise->get_future(); + raft_result->when_ready([sessions_closing_done_promise = std::move(sessions_closing_done_promise)]( + nuraft::cmd_result> & /*result*/, + nuraft::ptr & /*exception*/) { sessions_closing_done_promise->set_value(); }); + + auto session_shutdown_timeout = configuration_and_settings->coordination_settings->session_shutdown_timeout.totalMilliseconds(); + if (sessions_closing_done.wait_for(std::chrono::milliseconds(session_shutdown_timeout)) != std::future_status::ready) + LOG_WARNING( + log, + "Failed to close sessions in {}ms. If they are not closed, they will be closed after session timeout.", + session_shutdown_timeout); + } + + if (server) + server->shutdown(); + CurrentMetrics::set(CurrentMetrics::KeeperAliveConnections, 0); + } catch (...) { @@ -418,13 +463,15 @@ void KeeperDispatcher::sessionCleanerTask() LOG_INFO(log, "Found dead session {}, will try to close it", dead_session); /// Close session == send close request to raft server - Coordination::ZooKeeperRequestPtr request = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Close); + auto request = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Close); request->xid = Coordination::CLOSE_XID; - KeeperStorage::RequestForSession request_info; - request_info.request = request; using namespace std::chrono; - request_info.time = duration_cast(system_clock::now().time_since_epoch()).count(); - request_info.session_id = dead_session; + KeeperStorage::RequestForSession request_info + { + .session_id = dead_session, + .time = duration_cast(system_clock::now().time_since_epoch()).count(), + .request = std::move(request), + }; { std::lock_guard lock(push_request_mutex); if (!requests_queue->push(std::move(request_info))) diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index d311dfbd679..42d7d967b1f 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -442,9 +442,9 @@ void KeeperServer::shutdownRaftServer() void KeeperServer::shutdown() { - state_machine->shutdownStorage(); state_manager->flushAndShutDownLogStore(); shutdownRaftServer(); + state_machine->shutdownStorage(); } namespace @@ -520,7 +520,7 @@ bool KeeperServer::isFollower() const bool KeeperServer::isLeaderAlive() const { - return raft_instance->is_leader_alive(); + return raft_instance && raft_instance->is_leader_alive(); } /// TODO test whether taking failed peer in count @@ -617,7 +617,9 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ auto & entry_buf = entry->get_buf(); auto request_for_session = state_machine->parseRequest(entry_buf); request_for_session.zxid = next_zxid; - state_machine->preprocess(request_for_session); + if (!state_machine->preprocess(request_for_session)) + return nuraft::cb_func::ReturnCode::ReturnNull; + request_for_session.digest = state_machine->getNodesDigest(); entry = nuraft::cs_new(entry->get_term(), getZooKeeperLogEntry(request_for_session), entry->get_val_type()); break; diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index 3d6c80b5e41..c5a66ce29ca 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -191,12 +191,16 @@ KeeperStorage::RequestForSession KeeperStateMachine::parseRequest(nuraft::buffer return request_for_session; } -void KeeperStateMachine::preprocess(const KeeperStorage::RequestForSession & request_for_session) +bool KeeperStateMachine::preprocess(const KeeperStorage::RequestForSession & request_for_session) { if (request_for_session.request->getOpNum() == Coordination::OpNum::SessionID) - return; + return true; std::lock_guard lock(storage_and_responses_lock); + + if (storage->isFinalized()) + return false; + try { storage->preprocessRequest( @@ -215,6 +219,8 @@ void KeeperStateMachine::preprocess(const KeeperStorage::RequestForSession & req if (keeper_context->digest_enabled && request_for_session.digest) assertDigest(*request_for_session.digest, storage->getNodesDigest(false), *request_for_session.request, false); + + return true; } nuraft::ptr KeeperStateMachine::commit(const uint64_t log_idx, nuraft::buffer & data) diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index 9ddc4372d3b..fbd4fdc5ac2 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -33,7 +33,7 @@ public: static KeeperStorage::RequestForSession parseRequest(nuraft::buffer & data); - void preprocess(const KeeperStorage::RequestForSession & request_for_session); + bool preprocess(const KeeperStorage::RequestForSession & request_for_session); nuraft::ptr pre_commit(uint64_t log_idx, nuraft::buffer & data) override; diff --git a/src/Coordination/KeeperStateManager.cpp b/src/Coordination/KeeperStateManager.cpp index 3d7f5f2fb34..9b6aab5533e 100644 --- a/src/Coordination/KeeperStateManager.cpp +++ b/src/Coordination/KeeperStateManager.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -94,6 +95,14 @@ KeeperStateManager::parseServersConfiguration(const Poco::Util::AbstractConfigur continue; std::string full_prefix = config_prefix + ".raft_configuration." + server_key; + + if (getMultipleValuesFromConfig(config, full_prefix, "id").size() > 1 + || getMultipleValuesFromConfig(config, full_prefix, "hostname").size() > 1 + || getMultipleValuesFromConfig(config, full_prefix, "port").size() > 1) + { + throw Exception(ErrorCodes::RAFT_ERROR, "Multiple or or specified for a single "); + } + int new_server_id = config.getInt(full_prefix + ".id"); std::string hostname = config.getString(full_prefix + ".hostname"); int port = config.getInt(full_prefix + ".port"); diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index cfe614e1287..2328bc185a1 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1,11 +1,11 @@ #include #include -#include #include #include #include -#include #include + +#include #include #include #include @@ -15,8 +15,11 @@ #include #include #include + #include #include +#include + #include #include #include @@ -36,17 +39,6 @@ namespace ErrorCodes namespace { -String base64Encode(const String & decoded) -{ - std::ostringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - ostr.exceptions(std::ios::failbit); - Poco::Base64Encoder encoder(ostr); - encoder.rdbuf()->setLineLength(0); - encoder << decoded; - encoder.close(); - return ostr.str(); -} - String getSHA1(const String & userdata) { Poco::SHA1Engine engine; @@ -516,7 +508,7 @@ void KeeperStorage::UncommittedState::rollback(int64_t rollback_zxid) std::shared_ptr KeeperStorage::UncommittedState::getNode(StringRef path) const { - if (auto node_it = nodes.find(std::string{path}); node_it != nodes.end()) + if (auto node_it = nodes.find(path.toView()); node_it != nodes.end()) return node_it->second.node; return tryGetNodeFromStorage(path); @@ -524,7 +516,7 @@ std::shared_ptr KeeperStorage::UncommittedState::getNode(St Coordination::ACLs KeeperStorage::UncommittedState::getACLs(StringRef path) const { - if (auto node_it = nodes.find(std::string{path}); node_it != nodes.end()) + if (auto node_it = nodes.find(path.toView()); node_it != nodes.end()) return node_it->second.acls; auto node_it = storage.container.find(path); @@ -830,7 +822,9 @@ bool KeeperStorage::checkACL(StringRef path, int32_t permission, int64_t session void KeeperStorage::unregisterEphemeralPath(int64_t session_id, const std::string & path) { auto ephemerals_it = ephemerals.find(session_id); - assert(ephemerals_it != ephemerals.end()); + if (ephemerals_it == ephemerals.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Session {} is missing ephemeral path"); + ephemerals_it->second.erase(path); if (ephemerals_it->second.empty()) ephemerals.erase(ephemerals_it); @@ -1825,6 +1819,11 @@ void KeeperStorage::finalize() session_expiry_queue.clear(); } +bool KeeperStorage::isFinalized() const +{ + return finalized; +} + class KeeperStorageRequestProcessorsFactory final : private boost::noncopyable { @@ -1892,7 +1891,7 @@ UInt64 KeeperStorage::calculateNodesDigest(UInt64 current_digest, const std::vec if (!keeper_context->digest_enabled) return current_digest; - std::unordered_map> updated_nodes; + std::unordered_map> updated_nodes; for (const auto & delta : new_deltas) { diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index 73714771bf3..a40cca8e778 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -425,6 +425,8 @@ public: void finalize(); + bool isFinalized() const; + /// Set of methods for creating snapshots /// Turn on snapshot mode, so data inside Container is not deleted, but replaced with new version. diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index b7a33c4930d..29cd3c1c540 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -149,9 +149,9 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, CurrentMetrics::Met threads.resize(size_); for (auto & thread : threads) - thread = ThreadFromGlobalPool([this] { threadFunction(); }); + thread = ThreadFromGlobalPoolNoTracingContextPropagation([this] { threadFunction(); }); - delayed_thread = ThreadFromGlobalPool([this] { delayExecutionThreadFunction(); }); + delayed_thread = ThreadFromGlobalPoolNoTracingContextPropagation([this] { delayExecutionThreadFunction(); }); } @@ -168,7 +168,7 @@ void BackgroundSchedulePool::increaseThreadsCount(size_t new_threads_count) threads.resize(new_threads_count); for (size_t i = old_threads_count; i < new_threads_count; ++i) - threads[i] = ThreadFromGlobalPool([this] { threadFunction(); }); + threads[i] = ThreadFromGlobalPoolNoTracingContextPropagation([this] { threadFunction(); }); } diff --git a/src/Core/BackgroundSchedulePool.h b/src/Core/BackgroundSchedulePool.h index 36cbad145c9..1001d98e643 100644 --- a/src/Core/BackgroundSchedulePool.h +++ b/src/Core/BackgroundSchedulePool.h @@ -57,7 +57,9 @@ public: ~BackgroundSchedulePool(); private: - using Threads = std::vector; + /// BackgroundSchedulePool schedules a task on its own task queue, there's no need to construct/restore tracing context on this level. + /// This is also how ThreadPool class treats the tracing context. See ThreadPool for more information. + using Threads = std::vector; void threadFunction(); void delayExecutionThreadFunction(); @@ -83,7 +85,7 @@ private: std::condition_variable delayed_tasks_cond_var; std::mutex delayed_tasks_mutex; /// Thread waiting for next delayed task. - ThreadFromGlobalPool delayed_thread; + ThreadFromGlobalPoolNoTracingContextPropagation delayed_thread; /// Tasks ordered by scheduled time. DelayedTasks delayed_tasks; diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 3b7595eb886..33691e83d27 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -623,6 +623,7 @@ NamesAndTypesList Block::getNamesAndTypesList() const NamesAndTypes Block::getNamesAndTypes() const { NamesAndTypes res; + res.reserve(columns()); for (const auto & elem : data) res.emplace_back(elem.name, elem.type); diff --git a/src/Core/Field.h b/src/Core/Field.h index a0945b8315a..2924ed9f174 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -105,10 +105,6 @@ template bool decimalEqual(T x, T y, UInt32 x_scale, UInt32 y_scale template bool decimalLess(T x, T y, UInt32 x_scale, UInt32 y_scale); template bool decimalLessOrEqual(T x, T y, UInt32 x_scale, UInt32 y_scale); -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif template class DecimalField { @@ -168,9 +164,6 @@ private: T dec; UInt32 scale; }; -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif template constexpr bool is_decimal_field = false; template <> constexpr inline bool is_decimal_field> = true; @@ -432,16 +425,6 @@ public: bool isNegativeInfinity() const { return which == Types::Null && get().isNegativeInfinity(); } bool isPositiveInfinity() const { return which == Types::Null && get().isPositiveInfinity(); } - template - T & reinterpret(); - - template - const T & reinterpret() const - { - auto * mutable_this = const_cast *>(this); - return mutable_this->reinterpret(); - } - template bool tryGet(T & result) { const Types::Which requested = TypeToEnum>::value; @@ -559,7 +542,7 @@ public: case Types::Float64: { // Compare as UInt64 so that NaNs compare as equal. - return reinterpret() == rhs.reinterpret(); + return std::bit_cast(get()) == std::bit_cast(rhs.get()); } case Types::UUID: return get() == rhs.get(); case Types::String: return get() == rhs.get(); @@ -594,11 +577,6 @@ public: switch (field.which) { case Types::Null: return f(field.template get()); -// gcc 8.2.1 -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif case Types::UInt64: return f(field.template get()); case Types::UInt128: return f(field.template get()); case Types::UInt256: return f(field.template get()); @@ -622,9 +600,6 @@ public: case Types::Decimal128: return f(field.template get>()); case Types::Decimal256: return f(field.template get>()); case Types::AggregateFunctionState: return f(field.template get()); -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif } __builtin_unreachable(); @@ -858,30 +833,6 @@ auto & Field::safeGet() } -template -T & Field::reinterpret() -{ - assert(which != Types::String); // See specialization for char - using ValueType = std::decay_t; - ValueType * MAY_ALIAS ptr = reinterpret_cast(&storage); - return *ptr; -} - -// Specialize reinterpreting to char (used in ColumnUnique) to make sure Strings are reinterpreted correctly -// inline to avoid multiple definitions -template <> -inline char & Field::reinterpret() -{ - if (which == Types::String) - { - // For String we want to return a pointer to the data, not the start of the class - // as the layout of std::string depends on the STD version and options - char * ptr = reinterpret_cast(&storage)->data(); - return *ptr; - } - return *reinterpret_cast(&storage); -} - template Field::Field(T && rhs, enable_if_not_field_or_bool_or_stringlike_t) //-V730 { diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ec8b0785912..275a15f3ee6 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -86,6 +86,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(UInt64, s3_upload_part_size_multiply_parts_count_threshold, 1000, "Each time this number of parts was uploaded to S3 s3_min_upload_part_size multiplied by s3_upload_part_size_multiply_factor.", 0) \ M(UInt64, s3_max_single_part_upload_size, 32*1024*1024, "The maximum size of object to upload using singlepart upload to S3.", 0) \ M(UInt64, s3_max_single_read_retries, 4, "The maximum number of retries during single S3 read.", 0) \ + M(UInt64, s3_max_unexpected_write_error_retries, 4, "The maximum number of retries in case of unexpected errors during S3 write.", 0) \ M(UInt64, s3_max_redirects, 10, "Max number of S3 redirects hops allowed.", 0) \ M(UInt64, s3_max_connections, 1024, "The maximum number of connections per server.", 0) \ M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ @@ -136,6 +137,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ M(UInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is consumed. 0 means - same as 'max_threads'.", 0) \ M(Bool, enable_positional_arguments, true, "Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \ + M(Bool, enable_extended_results_for_datetime_functions, false, "Enable date functions like toLastDayOfMonth return Date32 results (instead of Date results) for Date32/DateTime64 arguments.", 0) \ \ M(Bool, group_by_use_nulls, false, "Treat columns mentioned in ROLLUP, CUBE or GROUPING SETS as Nullable", 0) \ \ @@ -351,7 +353,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(UInt64, max_ast_elements, 50000, "Maximum size of query syntax tree in number of nodes. Checked after parsing.", 0) \ M(UInt64, max_expanded_ast_elements, 500000, "Maximum size of query syntax tree in number of nodes after expansion of aliases and the asterisk.", 0) \ \ - M(UInt64, readonly, 0, "0 - everything is allowed. 1 - only read requests. 2 - only read requests, as well as changing settings, except for the 'readonly' setting.", 0) \ + M(UInt64, readonly, 0, "0 - no read-only restrictions. 1 - only read requests, as well as changing explicitly allowed settings. 2 - only read requests, as well as changing settings, except for the 'readonly' setting.", 0) \ \ M(UInt64, max_rows_in_set, 0, "Maximum size of the set (in number of elements) resulting from the execution of the IN section.", 0) \ M(UInt64, max_bytes_in_set, 0, "Maximum size of the set (in bytes in memory) resulting from the execution of the IN section.", 0) \ @@ -481,7 +483,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, optimize_if_chain_to_multiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \ M(Bool, optimize_multiif_to_if, true, "Replace 'multiIf' with only one condition to 'if'.", 0) \ M(Bool, optimize_if_transform_strings_to_enum, false, "Replaces string-type arguments in If and Transform to enum. Disabled by default cause it could make inconsistent change in distributed query that would lead to its fail.", 0) \ - M(Bool, optimize_monotonous_functions_in_order_by, true, "Replace monotonous function with its argument in ORDER BY", 0) \ + M(Bool, optimize_monotonous_functions_in_order_by, false, "Replace monotonous function with its argument in ORDER BY", 0) \ M(Bool, optimize_functions_to_subcolumns, false, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \ M(Bool, optimize_using_constraints, false, "Use constraints for query optimization", 0) \ M(Bool, optimize_substitute_columns, false, "Use constraints for column substitution", 0) \ @@ -527,7 +529,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, describe_extend_object_types, false, "Deduce concrete type of columns of type Object in DESCRIBE query", 0) \ M(Bool, describe_include_subcolumns, false, "If true, subcolumns of all table columns will be included into result of DESCRIBE query", 0) \ \ - M(Bool, optimize_rewrite_sum_if_to_count_if, true, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \ + M(Bool, optimize_rewrite_sum_if_to_count_if, false, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \ M(UInt64, insert_shard_id, 0, "If non zero, when insert into a distributed table, the data will be inserted into the shard `insert_shard_id` synchronously. Possible values range from 1 to `shards_number` of corresponding distributed table", 0) \ \ M(Bool, collect_hash_table_stats_during_aggregation, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \ @@ -602,6 +604,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, skip_download_if_exceeds_query_cache, true, "Skip download from remote filesystem if exceeds query cache size", 0) \ M(UInt64, max_query_cache_size, (128UL * 1024 * 1024 * 1024), "Max remote filesystem cache size that can be used by a single query", 0) \ \ + M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \ + \ M(Bool, use_structure_from_insertion_table_in_table_functions, false, "Use structure from insertion table instead of schema inference from data", 0) \ \ M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \ @@ -616,6 +620,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, allow_deprecated_database_ordinary, false, "Allow to create databases with deprecated Ordinary engine", 0) \ M(Bool, allow_deprecated_syntax_for_merge_tree, false, "Allow to create *MergeTree tables with deprecated engine definition syntax", 0) \ \ + M(Bool, force_grouping_standard_compatibility, true, "Make GROUPING function to return 1 when argument is not used as an aggregation key", 0) \ + \ M(Bool, schema_inference_use_cache_for_file, true, "Use cache in schema inference while using file table function", 0) \ M(Bool, schema_inference_use_cache_for_s3, true, "Use cache in schema inference while using s3 table function", 0) \ M(Bool, schema_inference_use_cache_for_hdfs, true, "Use cache in schema inference while using hdfs table function", 0) \ @@ -773,6 +779,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) \ M(UInt64, input_format_allow_errors_num, 0, "Maximum absolute amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \ M(Float, input_format_allow_errors_ratio, 0, "Maximum relative amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \ + M(String, input_format_record_errors_file_path, "", "Path of the file used to record errors while reading text formats (CSV, TSV).", 0) \ + M(String, errors_output_format, "CSV", "Method to write Errors to text output.", 0) \ \ M(String, format_schema, "", "Schema identifier (used by schema-based formats)", 0) \ M(String, format_template_resultset, "", "Path to file which contains format string for result set (for Template format)", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index be2def2c01a..b78b812da86 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -78,6 +78,7 @@ namespace SettingsChangesHistory /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) static std::map settings_changes_history = { + {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}}, {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"}, {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"}, {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}}, diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index 0025e44b489..20a4bef8176 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -48,6 +48,8 @@ struct SortColumnDescription bool with_fill; FillColumnDescription fill_description; + SortColumnDescription() = default; + explicit SortColumnDescription( const std::string & column_name_, int direction_ = 1, diff --git a/src/Core/Types.h b/src/Core/Types.h index 92546d7d07a..0dfc089f144 100644 --- a/src/Core/Types.h +++ b/src/Core/Types.h @@ -42,11 +42,6 @@ struct Null } }; -/// Ignore strange gcc warning https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55776 -#if !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wshadow" -#endif /// @note Except explicitly described you should not assume on TypeIndex numbers and/or their orders in this enum. enum class TypeIndex { @@ -89,9 +84,6 @@ enum class TypeIndex Map, Object, }; -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif using UInt128 = ::UInt128; diff --git a/src/Core/examples/coro.cpp b/src/Core/examples/coro.cpp index 370820a228d..fbccc261e9d 100644 --- a/src/Core/examples/coro.cpp +++ b/src/Core/examples/coro.cpp @@ -14,7 +14,7 @@ namespace std // NOLINT(cert-dcl58-cpp) { - using namespace experimental::coroutines_v1; + using namespace experimental::coroutines_v1; // NOLINT(cert-dcl58-cpp) } #if __has_warning("-Wdeprecated-experimental-coroutine") diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 3cf557ec5bf..e5d8d05acb5 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -159,6 +160,16 @@ void convertObjectsToTuples(Block & block, const NamesAndTypesList & extended_st } } +void deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block) +{ + if (!storage_snapshot->object_columns.empty()) + { + auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects(); + auto storage_columns = storage_snapshot->getColumns(options); + convertObjectsToTuples(block, storage_columns); + } +} + static bool isPrefix(const PathInData::Parts & prefix, const PathInData::Parts & parts) { if (prefix.size() > parts.size()) @@ -442,15 +453,19 @@ using SubcolumnsTreeWithColumns = SubcolumnsTree; using Node = SubcolumnsTreeWithColumns::Node; /// Creates data type and column from tree of subcolumns. -ColumnWithTypeAndDimensions createTypeFromNode(const Node * node) +ColumnWithTypeAndDimensions createTypeFromNode(const Node & node) { auto collect_tuple_elemets = [](const auto & children) { + if (children.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create type from empty Tuple or Nested node"); + std::vector> tuple_elements; tuple_elements.reserve(children.size()); for (const auto & [name, child] : children) { - auto column = createTypeFromNode(child.get()); + assert(child); + auto column = createTypeFromNode(*child); tuple_elements.emplace_back(name, std::move(column)); } @@ -464,13 +479,13 @@ ColumnWithTypeAndDimensions createTypeFromNode(const Node * node) return std::make_tuple(std::move(tuple_names), std::move(tuple_columns)); }; - if (node->kind == Node::SCALAR) + if (node.kind == Node::SCALAR) { - return node->data; + return node.data; } - else if (node->kind == Node::NESTED) + else if (node.kind == Node::NESTED) { - auto [tuple_names, tuple_columns] = collect_tuple_elemets(node->children); + auto [tuple_names, tuple_columns] = collect_tuple_elemets(node.children); Columns offsets_columns; offsets_columns.reserve(tuple_columns[0].array_dimensions + 1); @@ -481,7 +496,7 @@ ColumnWithTypeAndDimensions createTypeFromNode(const Node * node) /// `k1 Array(Nested(k2 Int, k3 Int))` and k1 is marked as Nested /// and `k2` and `k3` has anonymous_array_level = 1 in that case. - const auto & current_array = assert_cast(*node->data.column); + const auto & current_array = assert_cast(*node.data.column); offsets_columns.push_back(current_array.getOffsetsPtr()); auto first_column = tuple_columns[0].column; @@ -518,7 +533,7 @@ ColumnWithTypeAndDimensions createTypeFromNode(const Node * node) } else { - auto [tuple_names, tuple_columns] = collect_tuple_elemets(node->children); + auto [tuple_names, tuple_columns] = collect_tuple_elemets(node.children); size_t num_elements = tuple_columns.size(); Columns tuple_elements_columns(num_elements); @@ -576,6 +591,15 @@ std::pair unflattenObjectToTuple(const ColumnObject & co { const auto & subcolumns = column.getSubcolumns(); + if (subcolumns.empty()) + { + auto type = std::make_shared( + DataTypes{std::make_shared()}, + Names{ColumnObject::COLUMN_NAME_DUMMY}); + + return {type->createColumn()->cloneResized(column.size()), type}; + } + PathsInData paths; DataTypes types; Columns columns; @@ -602,6 +626,9 @@ std::pair unflattenTuple( assert(paths.size() == tuple_types.size()); assert(paths.size() == tuple_columns.size()); + if (paths.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot unflatten empty Tuple"); + /// We add all paths to the subcolumn tree and then create a type from it. /// The tree stores column, type and number of array dimensions /// for each intermediate node. diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 2dde0ed3e65..c60d5bec208 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -11,6 +11,9 @@ namespace DB { +struct StorageSnapshot; +using StorageSnapshotPtr = std::shared_ptr; + /// Returns number of dimensions in Array type. 0 if type is not array. size_t getNumberOfDimensions(const IDataType & type); @@ -38,6 +41,7 @@ DataTypePtr getDataTypeByColumn(const IColumn & column); /// Converts Object types and columns to Tuples in @columns_list and @block /// and checks that types are consistent with types in @extended_storage_columns. void convertObjectsToTuples(Block & block, const NamesAndTypesList & extended_storage_columns); +void deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block); /// Checks that each path is not the prefix of any other path. void checkObjectHasNoAmbiguosPaths(const PathsInData & paths); @@ -164,27 +168,24 @@ ColumnsDescription getObjectColumns( const ColumnsDescription & storage_columns, EntryColumnsGetter && entry_columns_getter) { - ColumnsDescription res; - - if (begin == end) - { - for (const auto & column : storage_columns) - { - if (isObject(column.type)) - { - auto tuple_type = std::make_shared( - DataTypes{std::make_shared()}, - Names{ColumnObject::COLUMN_NAME_DUMMY}); - - res.add({column.name, std::move(tuple_type)}); - } - } - - return res; - } - std::unordered_map types_in_entries; + /// Add dummy column for all Object columns + /// to not lose any column if it's missing + /// in all entries. If it exists in any entry + /// dummy column will be removed. + for (const auto & column : storage_columns) + { + if (isObject(column.type)) + { + auto tuple_type = std::make_shared( + DataTypes{std::make_shared()}, + Names{ColumnObject::COLUMN_NAME_DUMMY}); + + types_in_entries[column.name].push_back(std::move(tuple_type)); + } + } + for (auto it = begin; it != end; ++it) { const auto & entry_columns = entry_columns_getter(*it); @@ -196,6 +197,7 @@ ColumnsDescription getObjectColumns( } } + ColumnsDescription res; for (const auto & [name, types] : types_in_entries) res.add({name, getLeastCommonTypeForObject(types)}); diff --git a/src/DataTypes/Serializations/SerializationArray.cpp b/src/DataTypes/Serializations/SerializationArray.cpp index abd99038e98..eb93b5049a0 100644 --- a/src/DataTypes/Serializations/SerializationArray.cpp +++ b/src/DataTypes/Serializations/SerializationArray.cpp @@ -20,8 +20,13 @@ namespace ErrorCodes extern const int CANNOT_READ_ALL_DATA; extern const int CANNOT_READ_ARRAY_FROM_TEXT; extern const int LOGICAL_ERROR; + extern const int TOO_LARGE_ARRAY_SIZE; } +static constexpr size_t MAX_ARRAY_SIZE = 1ULL << 30; +static constexpr size_t MAX_ARRAYS_SIZE = 1ULL << 40; + + void SerializationArray::serializeBinary(const Field & field, WriteBuffer & ostr) const { const Array & a = field.get(); @@ -125,7 +130,12 @@ namespace { ColumnArray::Offset current_size = 0; readIntBinary(current_size, istr); - current_offset += current_size; + + if (unlikely(current_size > MAX_ARRAY_SIZE)) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array size is too large: {}", current_size); + if (unlikely(__builtin_add_overflow(current_offset, current_size, ¤t_offset))) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Deserialization of array offsets will lead to overflow"); + offset_values[i] = current_offset; ++i; } @@ -174,7 +184,7 @@ namespace { auto current_offset = offsets_data[i]; sizes_data[i] = current_offset - prev_offset; - prev_offset = current_offset; + prev_offset = current_offset; } return column_sizes; @@ -348,6 +358,9 @@ void SerializationArray::deserializeBinaryBulkWithMultipleStreams( throw Exception("Nested column is longer than last offset", ErrorCodes::LOGICAL_ERROR); size_t nested_limit = last_offset - nested_column->size(); + if (unlikely(nested_limit > MAX_ARRAYS_SIZE)) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array sizes are too large: {}", nested_limit); + /// Adjust value size hint. Divide it to the average array size. settings.avg_value_size_hint = nested_limit ? settings.avg_value_size_hint / nested_limit * offset_values.size() : 0; diff --git a/src/DataTypes/Serializations/SerializationFixedString.cpp b/src/DataTypes/Serializations/SerializationFixedString.cpp index 9baaf95cb52..dd476103108 100644 --- a/src/DataTypes/Serializations/SerializationFixedString.cpp +++ b/src/DataTypes/Serializations/SerializationFixedString.cpp @@ -24,6 +24,8 @@ namespace ErrorCodes extern const int TOO_LARGE_STRING_SIZE; } +static constexpr size_t MAX_STRINGS_SIZE = 1ULL << 30; + void SerializationFixedString::serializeBinary(const Field & field, WriteBuffer & ostr) const { const String & s = field.get(); @@ -85,8 +87,17 @@ void SerializationFixedString::deserializeBinaryBulk(IColumn & column, ReadBuffe ColumnFixedString::Chars & data = typeid_cast(column).getChars(); size_t initial_size = data.size(); - size_t max_bytes = limit * n; - data.resize(initial_size + max_bytes); + size_t max_bytes; + size_t new_data_size; + + if (unlikely(__builtin_mul_overflow(limit, n, &max_bytes))) + throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "Deserializing FixedString will lead to overflow"); + if (unlikely(max_bytes > MAX_STRINGS_SIZE)) + throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "Too large sizes of FixedString to deserialize: {}", max_bytes); + if (unlikely(__builtin_add_overflow(initial_size, max_bytes, &new_data_size))) + throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "Deserializing FixedString will lead to overflow"); + + data.resize(new_data_size); size_t read_bytes = istr.readBig(reinterpret_cast(&data[initial_size]), max_bytes); if (read_bytes % n != 0) diff --git a/src/DataTypes/Serializations/SerializationString.cpp b/src/DataTypes/Serializations/SerializationString.cpp index b6c4523eb52..042f2b3d45b 100644 --- a/src/DataTypes/Serializations/SerializationString.cpp +++ b/src/DataTypes/Serializations/SerializationString.cpp @@ -17,6 +17,7 @@ #include #endif + namespace DB { diff --git a/src/DataTypes/Serializations/SubcolumnsTree.h b/src/DataTypes/Serializations/SubcolumnsTree.h index f66f557bc8f..fda45e1e9a2 100644 --- a/src/DataTypes/Serializations/SubcolumnsTree.h +++ b/src/DataTypes/Serializations/SubcolumnsTree.h @@ -51,6 +51,8 @@ public: using NodeKind = typename Node::Kind; using NodePtr = std::shared_ptr; + SubcolumnsTree() : root(std::make_shared(Node::TUPLE)) {} + /// Add a leaf without any data in other nodes. bool add(const PathInData & path, const NodeData & leaf_data) { @@ -73,13 +75,9 @@ public: bool add(const PathInData & path, const NodeCreator & node_creator) { const auto & parts = path.getParts(); - if (parts.empty()) return false; - if (!root) - root = std::make_shared(Node::TUPLE); - Node * current_node = root.get(); for (size_t i = 0; i < parts.size() - 1; ++i) { @@ -166,13 +164,13 @@ public: return node; } - bool empty() const { return root == nullptr; } + bool empty() const { return root->children.empty(); } size_t size() const { return leaves.size(); } using Nodes = std::vector; const Nodes & getLeaves() const { return leaves; } - const Node * getRoot() const { return root.get(); } + const Node & getRoot() const { return *root; } using iterator = typename Nodes::iterator; using const_iterator = typename Nodes::const_iterator; @@ -186,11 +184,11 @@ public: private: const Node * findImpl(const PathInData & path, bool find_exact) const { - if (!root) + if (empty()) return nullptr; const auto & parts = path.getParts(); - const Node * current_node = root.get(); + const auto * current_node = root.get(); for (const auto & part : parts) { diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index b5bb6c7c759..796142884a3 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -185,6 +185,7 @@ void DatabaseOnDisk::createTable( if (create.attach_short_syntax) { /// Metadata already exists, table was detached + assert(fs::exists(getObjectMetadataPath(table_name))); removeDetachedPermanentlyFlag(local_context, table_name, table_metadata_path, true); attachTable(local_context, table_name, table, getTableDataPath(create)); return; diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index c6ef3867b63..f1bf56e2beb 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -1259,4 +1259,24 @@ void DatabaseReplicated::createTableRestoredFromBackup( } } +bool DatabaseReplicated::shouldReplicateQuery(const ContextPtr & query_context, const ASTPtr & query_ptr) const +{ + if (query_context->getClientInfo().is_replicated_database_internal) + return false; + + /// Some ALTERs are not replicated on database level + if (const auto * alter = query_ptr->as()) + { + return !alter->isAttachAlter() && !alter->isFetchAlter() && !alter->isDropPartitionAlter(); + } + + /// DROP DATABASE is not replicated + if (const auto * drop = query_ptr->as()) + { + return drop->table.get(); + } + + return true; +} + } diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 3aeec42b271..0c9a3b77844 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -46,7 +46,7 @@ public: /// Try to execute DLL query on current host as initial query. If query is succeed, /// then it will be executed on all replicas. - BlockIO tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, bool internal = false); + BlockIO tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, bool internal) override; bool hasReplicationThread() const override { return true; } @@ -75,6 +75,8 @@ public: std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; + bool shouldReplicateQuery(const ContextPtr & query_context, const ASTPtr & query_ptr) const override; + friend struct DatabaseReplicatedTask; friend class DatabaseReplicatedDDLWorker; private: diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 65dfb80846d..a0bb01a9550 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -338,6 +339,13 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not run a replication thread!", getEngineName()); } + virtual bool shouldReplicateQuery(const ContextPtr & /*query_context*/, const ASTPtr & /*query_ptr*/) const { return false; } + + virtual BlockIO tryEnqueueReplicatedDDL(const ASTPtr & /*query*/, ContextPtr /*query_context*/, [[maybe_unused]] bool internal = false) /// NOLINT + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not have replicated DDL queue", getEngineName()); + } + /// Returns CREATE TABLE queries and corresponding tables prepared for writing to a backup. virtual std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & context) const; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 81f33b27056..de7b9181533 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -71,6 +71,7 @@ public: virtual const String & getName() const = 0; /// Reserve the specified number of bytes. + /// Returns valid reservation or nullptr when failure. virtual ReservationPtr reserve(UInt64 bytes) = 0; virtual ~Space() = default; diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 21ed465e205..da7f8c871cb 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -118,7 +118,11 @@ void CachedOnDiskReadBufferFromFile::initialize(size_t offset, size_t size) } else { - file_segments_holder.emplace(cache->getOrSet(cache_key, offset, size, is_persistent)); + CreateFileSegmentSettings create_settings{ + .is_persistent = is_persistent + }; + + file_segments_holder.emplace(cache->getOrSet(cache_key, offset, size, create_settings)); } /** @@ -139,9 +143,11 @@ void CachedOnDiskReadBufferFromFile::initialize(size_t offset, size_t size) } CachedOnDiskReadBufferFromFile::ImplementationBufferPtr -CachedOnDiskReadBufferFromFile::getCacheReadBuffer(size_t offset) const +CachedOnDiskReadBufferFromFile::getCacheReadBuffer(const FileSegment & file_segment) const { - auto path = cache->getPathInLocalCache(cache_key, offset, is_persistent); + /// Use is_persistent flag from in-memory state of the filesegment, + /// because it is consistent with what is written on disk. + auto path = file_segment.getPathInLocalCache(); ReadSettings local_read_settings{settings}; /// Do not allow to use asynchronous version of LocalFSReadMethod. @@ -156,7 +162,7 @@ CachedOnDiskReadBufferFromFile::getCacheReadBuffer(size_t offset) const } CachedOnDiskReadBufferFromFile::ImplementationBufferPtr -CachedOnDiskReadBufferFromFile::getRemoteFSReadBuffer(FileSegmentPtr & file_segment, ReadType read_type_) +CachedOnDiskReadBufferFromFile::getRemoteFSReadBuffer(FileSegment & file_segment, ReadType read_type_) { switch (read_type_) { @@ -178,7 +184,7 @@ CachedOnDiskReadBufferFromFile::getRemoteFSReadBuffer(FileSegmentPtr & file_segm * Implementation buffer from segment1 is passed to segment2 once segment1 is loaded. */ - auto remote_fs_segment_reader = file_segment->getRemoteFileReader(); + auto remote_fs_segment_reader = file_segment.getRemoteFileReader(); if (!remote_fs_segment_reader) { @@ -189,7 +195,7 @@ CachedOnDiskReadBufferFromFile::getRemoteFSReadBuffer(FileSegmentPtr & file_segm ErrorCodes::CANNOT_USE_CACHE, "Cache cannot be used with a ReadBuffer which does not support right bounded reads"); - file_segment->setRemoteFileReader(remote_fs_segment_reader); + file_segment.setRemoteFileReader(remote_fs_segment_reader); } return remote_fs_segment_reader; @@ -201,8 +207,8 @@ CachedOnDiskReadBufferFromFile::getRemoteFSReadBuffer(FileSegmentPtr & file_segm if (remote_file_reader && remote_file_reader->getFileOffsetOfBufferEnd() == file_offset_of_buffer_end) return remote_file_reader; - auto remote_fs_segment_reader = file_segment->extractRemoteFileReader(); - if (remote_fs_segment_reader) + auto remote_fs_segment_reader = file_segment.extractRemoteFileReader(); + if (remote_fs_segment_reader && file_offset_of_buffer_end == remote_fs_segment_reader->getFileOffsetOfBufferEnd()) remote_file_reader = remote_fs_segment_reader; else remote_file_reader = implementation_buffer_creator(); @@ -217,11 +223,22 @@ CachedOnDiskReadBufferFromFile::getRemoteFSReadBuffer(FileSegmentPtr & file_segm } } +bool CachedOnDiskReadBufferFromFile::canStartFromCache(size_t current_offset, const FileSegment & file_segment) +{ + /// segment{k} state: DOWNLOADING + /// cache: [______|___________ + /// ^ + /// first_non_downloaded_offset (in progress) + /// requested_range: [__________] + /// ^ + /// current_offset + size_t first_non_downloaded_offset = file_segment.getFirstNonDownloadedOffset(); + return first_non_downloaded_offset > current_offset; +} + CachedOnDiskReadBufferFromFile::ImplementationBufferPtr CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & file_segment) { - auto range = file_segment->range(); - auto download_state = file_segment->state(); LOG_TEST(log, "getReadBufferForFileSegment: {}", file_segment->getInfoForLog()); @@ -230,13 +247,13 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil if (download_state == FileSegment::State::DOWNLOADED) { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } else { LOG_DEBUG(log, "Bypassing cache because `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` option is used"); read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; - return getRemoteFSReadBuffer(file_segment, read_type); + return getRemoteFSReadBuffer(*file_segment, read_type); } } @@ -248,27 +265,22 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil { LOG_DEBUG(log, "Bypassing cache because file segment state is `SKIP_CACHE`"); read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; - return getRemoteFSReadBuffer(file_segment, read_type); + return getRemoteFSReadBuffer(*file_segment, read_type); } case FileSegment::State::DOWNLOADING: { - size_t download_offset = file_segment->getDownloadOffset(); - bool can_start_from_cache = download_offset > file_offset_of_buffer_end; - - /// If file segment is being downloaded but we can already read - /// from already downloaded part, do that. - if (can_start_from_cache) + if (canStartFromCache(file_offset_of_buffer_end, *file_segment)) { /// segment{k} state: DOWNLOADING /// cache: [______|___________ /// ^ - /// download_offset (in progress) + /// first_non_downloaded_offset (in progress) /// requested_range: [__________] /// ^ /// file_offset_of_buffer_end read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } download_state = file_segment->wait(); @@ -277,72 +289,61 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil case FileSegment::State::DOWNLOADED: { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } case FileSegment::State::EMPTY: case FileSegment::State::PARTIALLY_DOWNLOADED: { - if (file_segment->getDownloadOffset() > file_offset_of_buffer_end) + if (canStartFromCache(file_offset_of_buffer_end, *file_segment)) { /// segment{k} state: PARTIALLY_DOWNLOADED /// cache: [______|___________ /// ^ - /// download_offset (in progress) + /// first_non_downloaded_offset (in progress) /// requested_range: [__________] /// ^ /// file_offset_of_buffer_end read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } auto downloader_id = file_segment->getOrSetDownloader(); if (downloader_id == file_segment->getCallerId()) { - size_t download_offset = file_segment->getDownloadOffset(); - bool can_start_from_cache = download_offset > file_offset_of_buffer_end; - - LOG_TEST( - log, - "Current download offset: {}, file offset of buffer end: {}", - download_offset, file_offset_of_buffer_end); - - if (can_start_from_cache) + if (canStartFromCache(file_offset_of_buffer_end, *file_segment)) { /// segment{k} /// cache: [______|___________ /// ^ - /// download_offset + /// first_non_downloaded_offset /// requested_range: [__________] /// ^ /// file_offset_of_buffer_end read_type = ReadType::CACHED; file_segment->resetDownloader(); - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } - if (download_offset < file_offset_of_buffer_end) + if (file_segment->getCurrentWriteOffset() < file_offset_of_buffer_end) { /// segment{1} /// cache: [_____|___________ /// ^ - /// download_offset + /// current_write_offset /// requested_range: [__________] /// ^ /// file_offset_of_buffer_end - assert(file_offset_of_buffer_end > file_segment->getDownloadOffset()); - bytes_to_predownload = file_offset_of_buffer_end - file_segment->getDownloadOffset(); - assert(bytes_to_predownload < range.size()); + LOG_TEST(log, "Predownload. File segment info: {}", file_segment->getInfoForLog()); + chassert(file_offset_of_buffer_end > file_segment->getCurrentWriteOffset()); + bytes_to_predownload = file_offset_of_buffer_end - file_segment->getCurrentWriteOffset(); + chassert(bytes_to_predownload < file_segment->range().size()); } - download_offset = file_segment->getDownloadOffset(); - can_start_from_cache = download_offset > file_offset_of_buffer_end; - assert(!can_start_from_cache); - read_type = ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; - return getRemoteFSReadBuffer(file_segment, read_type); + return getRemoteFSReadBuffer(*file_segment, read_type); } download_state = file_segment->state(); @@ -350,13 +351,10 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil } case FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION: { - size_t download_offset = file_segment->getDownloadOffset(); - bool can_start_from_cache = download_offset > file_offset_of_buffer_end; - - if (can_start_from_cache) + if (canStartFromCache(file_offset_of_buffer_end, *file_segment)) { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(*file_segment); } else { @@ -364,7 +362,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil log, "Bypassing cache because file segment state is `PARTIALLY_DOWNLOADED_NO_CONTINUATION` and downloaded part already used"); read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; - return getRemoteFSReadBuffer(file_segment, read_type); + return getRemoteFSReadBuffer(*file_segment, read_type); } } } @@ -374,8 +372,8 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil CachedOnDiskReadBufferFromFile::ImplementationBufferPtr CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegmentPtr & file_segment) { - assert(!file_segment->isDownloader()); - assert(file_offset_of_buffer_end >= file_segment->range().left); + chassert(!file_segment->isDownloader()); + chassert(file_offset_of_buffer_end >= file_segment->range().left); auto range = file_segment->range(); bytes_to_predownload = 0; @@ -389,10 +387,10 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegmentPtr & file_se ProfileEvents::FileSegmentWaitReadBufferMicroseconds, watch.elapsedMicroseconds()); [[maybe_unused]] auto download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; - assert(download_current_segment == file_segment->isDownloader()); + chassert(download_current_segment == file_segment->isDownloader()); - assert(file_segment->range() == range); - assert(file_offset_of_buffer_end >= range.left && file_offset_of_buffer_end <= range.right); + chassert(file_segment->range() == range); + chassert(file_offset_of_buffer_end >= range.left && file_offset_of_buffer_end <= range.right); LOG_TEST( log, @@ -441,12 +439,12 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegmentPtr & file_se } case ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE: { - assert(file_segment->isDownloader()); + chassert(file_segment->isDownloader()); if (bytes_to_predownload) { - size_t download_offset = file_segment->getDownloadOffset(); - read_buffer_for_file_segment->seek(download_offset, SEEK_SET); + size_t current_write_offset = file_segment->getCurrentWriteOffset(); + read_buffer_for_file_segment->seek(current_write_offset, SEEK_SET); } else { @@ -456,18 +454,15 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegmentPtr & file_se assert(static_cast(read_buffer_for_file_segment->getFileOffsetOfBufferEnd()) == file_offset_of_buffer_end); } - auto download_offset = file_segment->getDownloadOffset(); - if (download_offset != static_cast(read_buffer_for_file_segment->getPosition())) + auto current_write_offset = file_segment->getCurrentWriteOffset(); + if (current_write_offset != static_cast(read_buffer_for_file_segment->getPosition())) { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Buffer's offsets mismatch; cached buffer offset: {}, download_offset: {}, " - "position: {}, implementation buffer remaining read range: {}, file segment info: {}", - file_offset_of_buffer_end, - download_offset, - read_buffer_for_file_segment->getPosition(), - read_buffer_for_file_segment->getRemainingReadRange().toString(), - file_segment->getInfoForLog()); + "Buffer's offsets mismatch. Cached buffer offset: {}, current_write_offset: {} implementation buffer offset: {}, " + "implementation buffer remaining range: {}, file segment info: {}", + file_offset_of_buffer_end, current_write_offset, read_buffer_for_file_segment->getPosition(), + read_buffer_for_file_segment->getRemainingReadRange().toString(), file_segment->getInfoForLog()); } break; @@ -488,7 +483,7 @@ bool CachedOnDiskReadBufferFromFile::completeFileSegmentAndGetNext() auto & file_segment = *file_segment_it; [[maybe_unused]] const auto & range = file_segment->range(); - assert(file_offset_of_buffer_end > range.right); + chassert(file_offset_of_buffer_end > range.right); LOG_TEST( log, @@ -499,10 +494,8 @@ bool CachedOnDiskReadBufferFromFile::completeFileSegmentAndGetNext() /// Do not hold pointer to file segment if it is not needed anymore /// so can become releasable and can be evicted from cache. - /// If the status of filesegment state is SKIP_CACHE, it will not be deleted. - /// It will be deleted from the cache when the holder is destructed. - if ((*file_segment_it)->state() != FileSegment::State::SKIP_CACHE) - file_segments_holder->file_segments.erase(file_segment_it); + file_segment->completeWithoutState(); + file_segments_holder->file_segments.erase(file_segment_it); if (current_file_segment_it == file_segments_holder->file_segments.end()) return false; @@ -545,8 +538,8 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegmentPtr & file_segment) /// download from offset a'' < a', but return buffer from offset a'. LOG_TEST(log, "Bytes to predownload: {}, caller_id: {}", bytes_to_predownload, FileSegment::getCallerId()); - assert(implementation_buffer->getFileOffsetOfBufferEnd() == file_segment->getDownloadOffset()); - size_t current_offset = file_segment->getDownloadOffset(); + chassert(implementation_buffer->getFileOffsetOfBufferEnd() == file_segment->getCurrentWriteOffset()); + size_t current_offset = file_segment->getCurrentWriteOffset(); const auto & current_range = file_segment->range(); while (true) @@ -572,7 +565,7 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegmentPtr & file_segment) "current download offset: {}, expected: {}, eof: {}", bytes_to_predownload, current_range.toString(), - file_segment->getDownloadOffset(), + file_segment->getCurrentWriteOffset(), file_offset_of_buffer_end, implementation_buffer->eof()); @@ -582,18 +575,20 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegmentPtr & file_segment) { nextimpl_working_buffer_offset = implementation_buffer->offset(); - auto download_offset = file_segment->getDownloadOffset(); - if (download_offset != static_cast(implementation_buffer->getPosition()) - || download_offset != file_offset_of_buffer_end) + auto current_write_offset = file_segment->getCurrentWriteOffset(); + if (current_write_offset != static_cast(implementation_buffer->getPosition()) + || current_write_offset != file_offset_of_buffer_end) + { throw Exception( ErrorCodes::LOGICAL_ERROR, "Buffer's offsets mismatch after predownloading; download offset: {}, " "cached buffer offset: {}, implementation buffer offset: {}, " "file segment info: {}", - download_offset, + current_write_offset, file_offset_of_buffer_end, implementation_buffer->getPosition(), file_segment->getInfoForLog()); + } } break; @@ -609,7 +604,7 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegmentPtr & file_segment) { LOG_TEST(log, "Left to predownload: {}, buffer size: {}", bytes_to_predownload, current_impl_buffer_size); - assert(file_segment->getDownloadOffset() == static_cast(implementation_buffer->getPosition())); + chassert(file_segment->getCurrentWriteOffset() == static_cast(implementation_buffer->getPosition())); bool success = writeCache(implementation_buffer->buffer().begin(), current_predownload_size, current_offset, *file_segment); if (success) @@ -635,7 +630,7 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegmentPtr & file_segment) /// segment{1} /// cache: [_____|___________ /// ^ - /// download_offset + /// current_write_offset /// requested_range: [__________] /// ^ /// file_offset_of_buffer_end @@ -649,17 +644,18 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegmentPtr & file_segment) bytes_to_predownload = 0; file_segment->completeWithState(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); - LOG_TEST(log, "Bypassing cache because space reservation failed"); + LOG_TEST(log, "Bypassing cache because for {}", file_segment->getInfoForLog()); + read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; swap(*implementation_buffer); resetWorkingBuffer(); - implementation_buffer = getRemoteFSReadBuffer(file_segment, read_type); + implementation_buffer = getRemoteFSReadBuffer(*file_segment, read_type); swap(*implementation_buffer); - implementation_buffer->setReadUntilPosition(current_range.right + 1); /// [..., range.right] + implementation_buffer->setReadUntilPosition(file_segment->range().right + 1); /// [..., range.right] implementation_buffer->seek(file_offset_of_buffer_end, SEEK_SET); LOG_TEST( @@ -680,8 +676,8 @@ bool CachedOnDiskReadBufferFromFile::updateImplementationBufferIfNeeded() auto current_read_range = file_segment->range(); auto current_state = file_segment->state(); - assert(current_read_range.left <= file_offset_of_buffer_end); - assert(!file_segment->isDownloader()); + chassert(current_read_range.left <= file_offset_of_buffer_end); + chassert(!file_segment->isDownloader()); if (file_offset_of_buffer_end > current_read_range.right) { @@ -695,13 +691,15 @@ bool CachedOnDiskReadBufferFromFile::updateImplementationBufferIfNeeded() /// segment{k} /// cache: [______|___________ /// ^ - /// download_offset + /// current_write_offset /// requested_range: [__________] /// ^ /// file_offset_of_buffer_end - size_t download_offset = file_segment->getDownloadOffset(); - bool cached_part_is_finished = download_offset == file_offset_of_buffer_end; + auto current_write_offset = file_segment->getCurrentWriteOffset(); + bool cached_part_is_finished = current_write_offset == file_offset_of_buffer_end; + + LOG_TEST(log, "Current write offset: {}, file offset of buffer end: {}", current_write_offset, file_offset_of_buffer_end); if (cached_part_is_finished) { @@ -710,12 +708,12 @@ bool CachedOnDiskReadBufferFromFile::updateImplementationBufferIfNeeded() return true; } - else if (download_offset < file_offset_of_buffer_end) + else if (current_write_offset < file_offset_of_buffer_end) { throw Exception( ErrorCodes::LOGICAL_ERROR, "Expected {} >= {} ({})", - download_offset, file_offset_of_buffer_end, getInfoForLog()); + current_write_offset, file_offset_of_buffer_end, getInfoForLog()); } } @@ -725,7 +723,7 @@ bool CachedOnDiskReadBufferFromFile::updateImplementationBufferIfNeeded() * ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE means that on previous getImplementationBuffer() call * current buffer successfully called file_segment->getOrSetDownloader() and became a downloader * for this file segment. However, the downloader's term has a lifespan of 1 nextImpl() call, - * e.g. downloader reads buffer_size byte and calls completeBatchAndResetDownloader() and some other + * e.g. downloader reads buffer_size byte and calls completePartAndResetDownloader() and some other * thread can become a downloader if it calls getOrSetDownloader() faster. * * So downloader is committed to download only buffer_size bytes and then is not a downloader anymore, @@ -817,11 +815,11 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() if (need_complete_file_segment) { LOG_TEST(log, "Resetting downloader {} from scope exit", file_segment->getDownloader()); - file_segment->completeBatchAndResetDownloader(); + file_segment->completePartAndResetDownloader(); } } - assert(!file_segment->isDownloader()); + chassert(!file_segment->isDownloader()); } catch (...) { @@ -845,7 +843,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() (*current_file_segment_it)->incrementHitsCount(); } - assert(!internal_buffer.empty()); + chassert(!internal_buffer.empty()); swap(*implementation_buffer); @@ -854,15 +852,14 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() LOG_TEST( log, - "Current segment: {}, downloader: {}, current count: {}, position: {}, read range: {}", - current_read_range.toString(), - file_segment->getDownloader(), + "Current count: {}, position: {}, read range: {}, file segment: {}", implementation_buffer->count(), implementation_buffer->getPosition(), - implementation_buffer->getRemainingReadRange().toString()); + implementation_buffer->getRemainingReadRange().toString(), + file_segment->getInfoForLog()); - assert(current_read_range.left <= file_offset_of_buffer_end); - assert(current_read_range.right >= file_offset_of_buffer_end); + chassert(current_read_range.left <= file_offset_of_buffer_end); + chassert(current_read_range.right >= file_offset_of_buffer_end); bool result = false; size_t size = 0; @@ -939,24 +936,26 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() { if (download_current_segment) { - assert(file_offset_of_buffer_end + size - 1 <= file_segment->range().right); + chassert(file_offset_of_buffer_end + size - 1 <= file_segment->range().right); bool success = file_segment->reserve(size); if (success) { - assert(file_segment->getDownloadOffset() == static_cast(implementation_buffer->getPosition())); + chassert(file_segment->getCurrentWriteOffset() == static_cast(implementation_buffer->getPosition())); success = writeCache(implementation_buffer->position(), size, file_offset_of_buffer_end, *file_segment); if (success) { - assert(file_segment->getDownloadOffset() <= file_segment->range().right + 1); - assert( + chassert(file_segment->getCurrentWriteOffset() <= file_segment->range().right + 1); + chassert( std::next(current_file_segment_it) == file_segments_holder->file_segments.end() - || file_segment->getDownloadOffset() == implementation_buffer->getFileOffsetOfBufferEnd()); + || file_segment->getCurrentWriteOffset() == implementation_buffer->getFileOffsetOfBufferEnd()); + + LOG_TEST(log, "Successfully written {} bytes", size); } else { - assert(file_segment->state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); + chassert(file_segment->state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); LOG_TEST(log, "Bypassing cache because writeCache method failed"); } } @@ -984,7 +983,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() size_t remaining_size_to_read = std::min(current_read_range.right, read_until_position - 1) - file_offset_of_buffer_end + 1; size = std::min(size, remaining_size_to_read); - assert(implementation_buffer->buffer().size() >= nextimpl_working_buffer_offset + size); + chassert(implementation_buffer->buffer().size() >= nextimpl_working_buffer_offset + size); implementation_buffer->buffer().resize(nextimpl_working_buffer_offset + size); } @@ -996,15 +995,15 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() current_file_segment_counters.increment(ProfileEvents::FileSegmentUsedBytes, available()); if (download_current_segment) - file_segment->completeBatchAndResetDownloader(); + file_segment->completePartAndResetDownloader(); - assert(!file_segment->isDownloader()); + chassert(!file_segment->isDownloader()); LOG_TEST( log, "Key: {}. Returning with {} bytes, buffer position: {} (offset: {}, predownloaded: {}), " "buffer available: {}, current range: {}, current offset: {}, file segment state: {}, " - "download offset: {}, read_type: {}, reading until position: {}, started with offset: {}, " + "current write offset: {}, read_type: {}, reading until position: {}, started with offset: {}, " "remaining ranges: {}", getHexUIntLowercase(cache_key), working_buffer.size(), @@ -1015,7 +1014,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() current_read_range.toString(), file_offset_of_buffer_end, FileSegment::stateToString(file_segment->state()), - file_segment->getDownloadOffset(), + file_segment->getCurrentWriteOffset(), toString(read_type), read_until_position, first_offset, diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h index ed623272c12..535d01f3a8c 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h @@ -68,7 +68,7 @@ private: ImplementationBufferPtr getReadBufferForFileSegment(FileSegmentPtr & file_segment); - ImplementationBufferPtr getCacheReadBuffer(size_t offset) const; + ImplementationBufferPtr getCacheReadBuffer(const FileSegment & file_segment) const; std::optional getLastNonDownloadedOffset() const; @@ -80,7 +80,7 @@ private: void assertCorrectness() const; - std::shared_ptr getRemoteFSReadBuffer(FileSegmentPtr & file_segment, ReadType read_type_); + std::shared_ptr getRemoteFSReadBuffer(FileSegment & file_segment, ReadType read_type_); size_t getTotalSizeToRead(); @@ -90,6 +90,8 @@ private: bool writeCache(char * data, size_t size, size_t offset, FileSegment & file_segment); + static bool canStartFromCache(size_t current_offset, const FileSegment & file_segment); + Poco::Logger * log; FileCache::Key cache_key; String source_file_path; diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp index 56a21307c36..994bb743c5f 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp @@ -11,12 +11,16 @@ namespace ProfileEvents { extern const Event CachedWriteBufferCacheWriteBytes; extern const Event CachedWriteBufferCacheWriteMicroseconds; - extern const Event FileSegmentWriteMicroseconds; } namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace { class SwapHelper @@ -31,6 +35,178 @@ namespace }; } + +FileSegmentRangeWriter::FileSegmentRangeWriter( + FileCache * cache_, + const FileSegment::Key & key_, + std::shared_ptr cache_log_, + const String & query_id_, + const String & source_path_) + : cache(cache_) + , key(key_) + , cache_log(cache_log_) + , query_id(query_id_) + , source_path(source_path_) + , current_file_segment_it(file_segments_holder.file_segments.end()) +{ +} + +bool FileSegmentRangeWriter::write(const char * data, size_t size, size_t offset, bool is_persistent) +{ + if (finalized) + return false; + + auto & file_segments = file_segments_holder.file_segments; + + if (current_file_segment_it == file_segments.end()) + { + current_file_segment_it = allocateFileSegment(current_file_segment_write_offset, is_persistent); + } + else + { + auto file_segment = *current_file_segment_it; + assert(file_segment->getCurrentWriteOffset() == current_file_segment_write_offset); + + if (current_file_segment_write_offset != offset) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot write file segment at offset {}, because current write offset is: {}", + offset, current_file_segment_write_offset); + } + + if (file_segment->range().size() == file_segment->getDownloadedSize()) + { + completeFileSegment(*file_segment); + current_file_segment_it = allocateFileSegment(current_file_segment_write_offset, is_persistent); + } + } + + auto & file_segment = *current_file_segment_it; + + auto downloader = file_segment->getOrSetDownloader(); + if (downloader != FileSegment::getCallerId()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to set a downloader. ({})", file_segment->getInfoForLog()); + + SCOPE_EXIT({ + if (file_segment->isDownloader()) + file_segment->completePartAndResetDownloader(); + }); + + bool reserved = file_segment->reserve(size); + if (!reserved) + { + file_segment->completeWithState(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); + appendFilesystemCacheLog(*file_segment); + + LOG_DEBUG( + &Poco::Logger::get("FileSegmentRangeWriter"), + "Unsuccessful space reservation attempt (size: {}, file segment info: {}", + size, file_segment->getInfoForLog()); + + return false; + } + + try + { + file_segment->write(data, size, offset); + } + catch (...) + { + file_segment->completePartAndResetDownloader(); + throw; + } + + file_segment->completePartAndResetDownloader(); + current_file_segment_write_offset += size; + + return true; +} + +void FileSegmentRangeWriter::finalize() +{ + if (finalized) + return; + + auto & file_segments = file_segments_holder.file_segments; + if (file_segments.empty() || current_file_segment_it == file_segments.end()) + return; + + completeFileSegment(**current_file_segment_it); + finalized = true; +} + +FileSegmentRangeWriter::~FileSegmentRangeWriter() +{ + try + { + if (!finalized) + finalize(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } +} + +FileSegments::iterator FileSegmentRangeWriter::allocateFileSegment(size_t offset, bool is_persistent) +{ + /** + * Allocate a new file segment starting `offset`. + * File segment capacity will equal `max_file_segment_size`, but actual size is 0. + */ + + std::lock_guard cache_lock(cache->mutex); + + CreateFileSegmentSettings create_settings + { + .is_persistent = is_persistent, + }; + + /// We set max_file_segment_size to be downloaded, + /// if we have less size to write, file segment will be resized in complete() method. + auto file_segment = cache->createFileSegmentForDownload( + key, offset, cache->max_file_segment_size, create_settings, cache_lock); + + return file_segments_holder.add(std::move(file_segment)); +} + +void FileSegmentRangeWriter::appendFilesystemCacheLog(const FileSegment & file_segment) +{ + if (cache_log) + { + auto file_segment_range = file_segment.range(); + size_t file_segment_right_bound = file_segment_range.left + file_segment.getDownloadedSize() - 1; + + FilesystemCacheLogElement elem + { + .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()), + .query_id = query_id, + .source_file_path = source_path, + .file_segment_range = { file_segment_range.left, file_segment_right_bound }, + .requested_range = {}, + .cache_type = FilesystemCacheLogElement::CacheType::WRITE_THROUGH_CACHE, + .file_segment_size = file_segment_range.size(), + .read_from_cache_attempted = false, + .read_buffer_id = {}, + .profile_counters = nullptr, + }; + + cache_log->add(elem); + } +} + +void FileSegmentRangeWriter::completeFileSegment(FileSegment & file_segment) +{ + /// File segment can be detached if space reservation failed. + if (file_segment.isDetached()) + return; + + file_segment.completeWithoutState(); + appendFilesystemCacheLog(file_segment); +} + + CachedOnDiskWriteBufferFromFile::CachedOnDiskWriteBufferFromFile( std::unique_ptr impl_, FileCachePtr cache_, @@ -47,7 +223,6 @@ CachedOnDiskWriteBufferFromFile::CachedOnDiskWriteBufferFromFile( , is_persistent_cache_file(is_persistent_cache_file_) , query_id(query_id_) , enable_cache_log(!query_id_.empty() && settings_.enable_filesystem_cache_log) - , cache_log(Context::getGlobalContextInstance()->getFilesystemCacheLog()) { } @@ -77,25 +252,27 @@ void CachedOnDiskWriteBufferFromFile::nextImpl() void CachedOnDiskWriteBufferFromFile::cacheData(char * data, size_t size) { - if (stop_caching) + if (cache_in_error_state_or_disabled) return; if (!cache_writer) { - cache_writer = std::make_unique( - cache.get(), key, [this](const FileSegment & file_segment) { appendFilesystemCacheLog(file_segment); }); + std::shared_ptr cache_log; + if (enable_cache_log) + cache_log = Context::getGlobalContextInstance()->getFilesystemCacheLog(); + + cache_writer = std::make_unique(cache.get(), key, cache_log, query_id, source_path); } Stopwatch watch(CLOCK_MONOTONIC); + cache_in_error_state_or_disabled = true; + try { if (!cache_writer->write(data, size, current_download_offset, is_persistent_cache_file)) { LOG_INFO(log, "Write-through cache is stopped as cache limit is reached and nothing can be evicted"); - - /// No space left, disable caching. - stop_caching = true; return; } } @@ -120,33 +297,7 @@ void CachedOnDiskWriteBufferFromFile::cacheData(char * data, size_t size) ProfileEvents::increment(ProfileEvents::CachedWriteBufferCacheWriteBytes, size); ProfileEvents::increment(ProfileEvents::CachedWriteBufferCacheWriteMicroseconds, watch.elapsedMicroseconds()); - current_file_segment_counters.increment( - ProfileEvents::FileSegmentWriteMicroseconds, watch.elapsedMicroseconds()); -} - -void CachedOnDiskWriteBufferFromFile::appendFilesystemCacheLog(const FileSegment & file_segment) -{ - if (cache_log) - { - auto file_segment_range = file_segment.range(); - FilesystemCacheLogElement elem - { - .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()), - .query_id = query_id, - .source_file_path = source_path, - .file_segment_range = { file_segment_range.left, file_segment_range.right }, - .requested_range = {}, - .cache_type = FilesystemCacheLogElement::CacheType::WRITE_THROUGH_CACHE, - .file_segment_size = file_segment_range.size(), - .read_from_cache_attempted = false, - .read_buffer_id = {}, - .profile_counters = std::make_shared(current_file_segment_counters.getPartiallyAtomicSnapshot()), - }; - - current_file_segment_counters.reset(); - - cache_log->add(elem); - } + cache_in_error_state_or_disabled = false; } void CachedOnDiskWriteBufferFromFile::finalizeImpl() diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h index fa861fea14b..cec7305ab1b 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h @@ -13,11 +13,57 @@ class Logger; namespace DB { +/** +* We want to write eventually some size, which is not known until the very end. +* Therefore we allocate file segments lazily. Each file segment is assigned capacity +* of max_file_segment_size, but reserved_size remains 0, until call to tryReserve(). +* Once current file segment is full (reached max_file_segment_size), we allocate a +* new file segment. All allocated file segments resize in file segments holder. +* If at the end of all writes, the last file segment is not full, then it is resized. +*/ +class FileSegmentRangeWriter +{ +public: + FileSegmentRangeWriter( + FileCache * cache_, const FileSegment::Key & key_, + std::shared_ptr cache_log_, const String & query_id_, const String & source_path_); + + /** + * Write a range of file segments. Allocate file segment of `max_file_segment_size` and write to + * it until it is full and then allocate next file segment. + */ + bool write(const char * data, size_t size, size_t offset, bool is_persistent); + + void finalize(); + + ~FileSegmentRangeWriter(); + +private: + FileSegments::iterator allocateFileSegment(size_t offset, bool is_persistent); + + void appendFilesystemCacheLog(const FileSegment & file_segment); + + void completeFileSegment(FileSegment & file_segment); + + FileCache * cache; + FileSegment::Key key; + + std::shared_ptr cache_log; + String query_id; + String source_path; + + FileSegmentsHolder file_segments_holder{}; + FileSegments::iterator current_file_segment_it; + + size_t current_file_segment_write_offset = 0; + + bool finalized = false; +}; + + /** * Write buffer for filesystem caching on write operations. */ -class FileSegmentRangeWriter; - class CachedOnDiskWriteBufferFromFile final : public WriteBufferFromFileDecorator { public: @@ -36,7 +82,6 @@ public: private: void cacheData(char * data, size_t size); - void appendFilesystemCacheLog(const FileSegment & file_segment); Poco::Logger * log; @@ -49,11 +94,9 @@ private: const String query_id; bool enable_cache_log; - std::shared_ptr cache_log; - bool stop_caching = false; + bool cache_in_error_state_or_disabled = false; - ProfileEvents::Counters current_file_segment_counters; std::unique_ptr cache_writer; }; diff --git a/src/Disks/IO/ThreadPoolReader.cpp b/src/Disks/IO/ThreadPoolReader.cpp index 9b38607c204..d2b3bcbaa5e 100644 --- a/src/Disks/IO/ThreadPoolReader.cpp +++ b/src/Disks/IO/ThreadPoolReader.cpp @@ -108,8 +108,19 @@ std::future ThreadPoolReader::submit(Request reques if (has_pread_nowait_support.load(std::memory_order_relaxed)) { + /// It reports real time spent including the time spent while thread was preempted doing nothing. + /// And it is Ok for the purpose of this watch (it is used to lower the number of threads to read from tables). + /// Sometimes it is better to use taskstats::blkio_delay_total, but it is quite expensive to get it + /// (TaskStatsInfoGetter has about 500K RPS). Stopwatch watch(CLOCK_MONOTONIC); + SCOPE_EXIT({ + watch.stop(); + + ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHitElapsedMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); + }); + std::promise promise; std::future future = promise.get_future(); @@ -135,11 +146,6 @@ std::future ThreadPoolReader::submit(Request reques { /// The file has ended. promise.set_value({0, 0}); - - watch.stop(); - ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHitElapsedMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); - return future; } @@ -179,18 +185,10 @@ std::future ThreadPoolReader::submit(Request reques if (bytes_read) { - /// It reports real time spent including the time spent while thread was preempted doing nothing. - /// And it is Ok for the purpose of this watch (it is used to lower the number of threads to read from tables). - /// Sometimes it is better to use taskstats::blkio_delay_total, but it is quite expensive to get it - /// (TaskStatsInfoGetter has about 500K RPS). - watch.stop(); - /// Read successfully from page cache. ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHit); ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHitBytes, bytes_read); ProfileEvents::increment(ProfileEvents::ReadBufferFromFileDescriptorReadBytes, bytes_read); - ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHitElapsedMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); promise.set_value({bytes_read, request.ignore}); return future; @@ -226,6 +224,12 @@ std::future ThreadPoolReader::submit(Request reques setThreadName("ThreadPoolRead"); Stopwatch watch(CLOCK_MONOTONIC); + SCOPE_EXIT({ + watch.stop(); + + ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheMissElapsedMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); + }); size_t bytes_read = 0; while (!bytes_read) @@ -254,8 +258,6 @@ std::future ThreadPoolReader::submit(Request reques ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheMissBytes, bytes_read); ProfileEvents::increment(ProfileEvents::ReadBufferFromFileDescriptorReadBytes, bytes_read); - ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheMissElapsedMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); return Result{ .size = bytes_read, .offset = request.ignore }; }); diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index db8f90e777d..f4462a0f8e6 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -156,6 +156,8 @@ void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std:: e.code() == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF || e.code() == ErrorCodes::CANNOT_READ_ALL_DATA) return; + + throw; } catch (const fs::filesystem_error & e) { diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp index f18debe8a8b..56cc20098ba 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp @@ -13,7 +13,6 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_FORMAT; - extern const int LOGICAL_ERROR; } void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf) @@ -131,9 +130,6 @@ DiskObjectStorageMetadata::DiskObjectStorageMetadata( void DiskObjectStorageMetadata::addObject(const String & path, size_t size) { - if (!object_storage_root_path.empty() && path.starts_with(object_storage_root_path)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected relative path"); - total_size += size; storage_objects.emplace_back(path, size); } diff --git a/src/Disks/ObjectStorages/IObjectStorage.cpp b/src/Disks/ObjectStorages/IObjectStorage.cpp index 1a128770015..65720ec3937 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.cpp +++ b/src/Disks/ObjectStorages/IObjectStorage.cpp @@ -61,6 +61,7 @@ ReadSettings IObjectStorage::patchSettings(const ReadSettings & read_settings) c std::unique_lock lock{throttlers_mutex}; ReadSettings settings{read_settings}; settings.remote_throttler = remote_read_throttler; + settings.for_object_storage = true; return settings; } @@ -69,6 +70,7 @@ WriteSettings IObjectStorage::patchSettings(const WriteSettings & write_settings std::unique_lock lock{throttlers_mutex}; WriteSettings settings{write_settings}; settings.remote_throttler = remote_write_throttler; + settings.for_object_storage = true; return settings; } diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.cpp index 39c1b020b3a..010fc103254 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.cpp @@ -324,11 +324,15 @@ void UnlinkMetadataFileOperation::execute(std::unique_lock & void UnlinkMetadataFileOperation::undo() { - if (write_operation) - write_operation->undo(); - + /// Operations MUST be reverted in the reversed order, so + /// when we apply operation #1 (write) and operation #2 (unlink) + /// we should revert #2 and only after it #1. Otherwise #1 will overwrite + /// file with incorrect data. if (unlink_operation) unlink_operation->undo(); + + if (write_operation) + write_operation->undo(); } void SetReadonlyFileOperation::execute(std::unique_lock & metadata_lock) diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 45304ac2fac..c2131a51b74 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1,4 +1,5 @@ #include +#include #if USE_AWS_S3 @@ -32,6 +33,27 @@ #include +namespace ProfileEvents +{ + extern const Event S3DeleteObjects; + extern const Event S3HeadObject; + extern const Event S3ListObjects; + extern const Event S3CopyObject; + extern const Event S3CreateMultipartUpload; + extern const Event S3UploadPartCopy; + extern const Event S3AbortMultipartUpload; + extern const Event S3CompleteMultipartUpload; + + extern const Event DiskS3DeleteObjects; + extern const Event DiskS3HeadObject; + extern const Event DiskS3ListObjects; + extern const Event DiskS3CopyObject; + extern const Event DiskS3CreateMultipartUpload; + extern const Event DiskS3UploadPartCopy; + extern const Event DiskS3AbortMultipartUpload; + extern const Event DiskS3CompleteMultipartUpload; +} + namespace DB { @@ -109,6 +131,9 @@ std::string S3ObjectStorage::generateBlobNameForPath(const std::string & /* path Aws::S3::Model::HeadObjectOutcome S3ObjectStorage::requestObjectHeadData(const std::string & bucket_from, const std::string & key) const { auto client_ptr = client.get(); + + ProfileEvents::increment(ProfileEvents::S3HeadObject); + ProfileEvents::increment(ProfileEvents::DiskS3HeadObject); Aws::S3::Model::HeadObjectRequest request; request.SetBucket(bucket_from); request.SetKey(key); @@ -232,6 +257,8 @@ void S3ObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize Aws::S3::Model::ListObjectsV2Outcome outcome; do { + ProfileEvents::increment(ProfileEvents::S3ListObjects); + ProfileEvents::increment(ProfileEvents::DiskS3ListObjects); outcome = client_ptr->ListObjectsV2(request); throwIfError(outcome); @@ -252,6 +279,8 @@ void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exis { auto client_ptr = client.get(); + ProfileEvents::increment(ProfileEvents::S3DeleteObjects); + ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); Aws::S3::Model::DeleteObjectRequest request; request.SetBucket(bucket); request.SetKey(object.absolute_path); @@ -297,6 +326,9 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e Aws::S3::Model::Delete delkeys; delkeys.SetObjects(current_chunk); + + ProfileEvents::increment(ProfileEvents::S3DeleteObjects); + ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); Aws::S3::Model::DeleteObjectsRequest request; request.SetBucket(bucket); request.SetDelete(delkeys); @@ -370,6 +402,9 @@ void S3ObjectStorage::copyObjectImpl( std::optional metadata) const { auto client_ptr = client.get(); + + ProfileEvents::increment(ProfileEvents::S3CopyObject); + ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); Aws::S3::Model::CopyObjectRequest request; request.SetCopySource(src_bucket + "/" + src_key); request.SetBucket(dst_bucket); @@ -418,6 +453,8 @@ void S3ObjectStorage::copyObjectMultipartImpl( String multipart_upload_id; { + ProfileEvents::increment(ProfileEvents::S3CreateMultipartUpload); + ProfileEvents::increment(ProfileEvents::DiskS3CreateMultipartUpload); Aws::S3::Model::CreateMultipartUploadRequest request; request.SetBucket(dst_bucket); request.SetKey(dst_key); @@ -436,6 +473,8 @@ void S3ObjectStorage::copyObjectMultipartImpl( size_t upload_part_size = settings_ptr->s3_settings.min_upload_part_size; for (size_t position = 0, part_number = 1; position < size; ++part_number, position += upload_part_size) { + ProfileEvents::increment(ProfileEvents::S3UploadPartCopy); + ProfileEvents::increment(ProfileEvents::DiskS3UploadPartCopy); Aws::S3::Model::UploadPartCopyRequest part_request; part_request.SetCopySource(src_bucket + "/" + src_key); part_request.SetBucket(dst_bucket); @@ -447,6 +486,8 @@ void S3ObjectStorage::copyObjectMultipartImpl( auto outcome = client_ptr->UploadPartCopy(part_request); if (!outcome.IsSuccess()) { + ProfileEvents::increment(ProfileEvents::S3AbortMultipartUpload); + ProfileEvents::increment(ProfileEvents::DiskS3AbortMultipartUpload); Aws::S3::Model::AbortMultipartUploadRequest abort_request; abort_request.SetBucket(dst_bucket); abort_request.SetKey(dst_key); @@ -461,6 +502,8 @@ void S3ObjectStorage::copyObjectMultipartImpl( } { + ProfileEvents::increment(ProfileEvents::S3CompleteMultipartUpload); + ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Aws::S3::Model::CompleteMultipartUploadRequest req; req.SetBucket(dst_bucket); req.SetKey(dst_key); diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index a93d95d91bd..b5efc11db8b 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -39,6 +39,7 @@ std::unique_ptr getSettings(const Poco::Util::AbstractC rw_settings.upload_part_size_multiply_parts_count_threshold = config.getUInt64(config_prefix + ".s3_upload_part_size_multiply_parts_count_threshold", context->getSettingsRef().s3_upload_part_size_multiply_parts_count_threshold); rw_settings.max_single_part_upload_size = config.getUInt64(config_prefix + ".s3_max_single_part_upload_size", context->getSettingsRef().s3_max_single_part_upload_size); rw_settings.check_objects_after_upload = config.getUInt64(config_prefix + ".s3_check_objects_after_upload", context->getSettingsRef().s3_check_objects_after_upload); + rw_settings.max_unexpected_write_error_retries = config.getUInt64(config_prefix + ".s3_max_unexpected_write_error_retries", context->getSettingsRef().s3_max_unexpected_write_error_retries); return std::make_unique( rw_settings, diff --git a/src/Disks/VolumeJBOD.h b/src/Disks/VolumeJBOD.h index 21d61e6dd8d..81da64c488d 100644 --- a/src/Disks/VolumeJBOD.h +++ b/src/Disks/VolumeJBOD.h @@ -82,6 +82,9 @@ private: ReservationPtr reserve(uint64_t bytes) { ReservationPtr reservation = disk->reserve(bytes); + if (!reservation) + return {}; + /// Not just subtract bytes, but update the value, /// since some reservations may be done directly via IDisk, or not by ClickHouse. free_size = reservation->getUnreservedSpace(); diff --git a/src/Disks/tests/gtest_disk.cpp b/src/Disks/tests/gtest_disk.cpp index 36f91249391..908e76b5c63 100644 --- a/src/Disks/tests/gtest_disk.cpp +++ b/src/Disks/tests/gtest_disk.cpp @@ -7,12 +7,6 @@ namespace fs = std::filesystem; -#if !defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wsuggest-override" -#endif - - template DB::DiskPtr createDisk(); diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 780b6bb6201..3de4a0de391 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -243,11 +243,20 @@ InputFormatPtr FormatFactory::getInput( ParallelParsingInputFormat::Params params{ buf, sample, parser_creator, file_segmentation_engine, name, settings.max_threads, settings.min_chunk_bytes_for_parallel_parsing, context->getApplicationType() == Context::ApplicationType::SERVER}; - return std::make_shared(params); + auto format = std::make_shared(params); + if (!settings.input_format_record_errors_file_path.toString().empty()) + { + format->setErrorsLogger(std::make_shared(context)); + } + return format; } auto format = getInputFormat(name, buf, sample, context, max_block_size, format_settings); + if (!settings.input_format_record_errors_file_path.toString().empty()) + { + format->setErrorsLogger(std::make_shared(context)); + } return format; } @@ -521,6 +530,7 @@ String FormatFactory::getFormatFromFileDescriptor(int fd) return getFormatFromFileName(file_path, false); return ""; #else + (void)fd; return ""; #endif } diff --git a/src/Formats/ReadSchemaUtils.cpp b/src/Formats/ReadSchemaUtils.cpp index d09cb9ff9ad..8468f540253 100644 --- a/src/Formats/ReadSchemaUtils.cpp +++ b/src/Formats/ReadSchemaUtils.cpp @@ -63,9 +63,10 @@ ColumnsDescription readSchemaFromFormat( { names_and_types = external_schema_reader->readSchema(); } - catch (const DB::Exception & e) + catch (Exception & e) { - throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot extract table structure from {} format file. Error: {}. You can specify the structure manually", format_name, e.message()); + e.addMessage(fmt::format("Cannot extract table structure from {} format file. You can specify the structure manually", format_name)); + throw; } } else if (FormatFactory::instance().checkIfFormatHasSchemaReader(format_name)) @@ -85,6 +86,12 @@ ColumnsDescription readSchemaFromFormat( break; is_eof = buf->eof(); } + catch (Exception & e) + { + e.addMessage(fmt::format( + "Cannot extract table structure from {} format file. You can specify the structure manually", format_name)); + throw; + } catch (...) { auto exception_message = getCurrentExceptionMessage(false); @@ -136,7 +143,21 @@ ColumnsDescription readSchemaFromFormat( } if (!retry || !isRetryableSchemaInferenceError(getCurrentExceptionCode())) - throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot extract table structure from {} format file. Error: {}. You can specify the structure manually", format_name, exception_message); + { + try + { + throw; + } + catch (Exception & e) + { + e.addMessage(fmt::format("Cannot extract table structure from {} format file. You can specify the structure manually", format_name)); + throw; + } + catch (...) + { + throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot extract table structure from {} format file. Error: {}. You can specify the structure manually", format_name, exception_message); + } + } exception_messages += "\n" + exception_message; } diff --git a/src/Functions/CustomWeekTransforms.h b/src/Functions/CustomWeekTransforms.h index 3378aec02d5..5fa51d5f5e0 100644 --- a/src/Functions/CustomWeekTransforms.h +++ b/src/Functions/CustomWeekTransforms.h @@ -82,6 +82,14 @@ struct ToStartOfWeekImpl { return time_zone.toFirstDayNumOfWeek(DayNum(d), week_mode); } + static inline Int64 execute_extended_result(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode); + } + static inline Int32 execute_extended_result(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode); + } using FactorTransform = ZeroTransform; }; @@ -115,7 +123,7 @@ struct ToWeekImpl using FactorTransform = ToStartOfYearImpl; }; -template +template struct WeekTransformer { explicit WeekTransformer(Transform transform_) @@ -130,7 +138,10 @@ struct WeekTransformer vec_to.resize(size); for (size_t i = 0; i < size; ++i) - vec_to[i] = transform.execute(vec_from[i], week_mode, time_zone); + if constexpr (is_extended_result) + vec_to[i] = transform.execute_extended_result(vec_from[i], week_mode, time_zone); + else + vec_to[i] = transform.execute(vec_from[i], week_mode, time_zone); } private: @@ -138,13 +149,13 @@ private: }; -template +template struct CustomWeekTransformImpl { template static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/, Transform transform = {}) { - const auto op = WeekTransformer{std::move(transform)}; + const auto op = WeekTransformer{std::move(transform)}; UInt8 week_mode = DEFAULT_WEEK_MODE; if (arguments.size() > 1) diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 66d57f2463f..fbe8e4bfcfe 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -161,7 +161,14 @@ struct ToMondayImpl { return time_zone.toFirstDayNumOfWeek(DayNum(d)); } - + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -185,6 +192,14 @@ struct ToStartOfMonthImpl { return time_zone.toFirstDayNumOfMonth(DayNum(d)); } + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -218,7 +233,14 @@ struct ToLastDayOfMonthImpl /// 0xFFF9 is Int value for 2149-05-31 -- the last day where we can actually find LastDayOfMonth. This will also be the return value. return time_zone.toLastDayNumOfMonth(DayNum(std::min(d, UInt16(0xFFF9)))); } - + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toLastDayNumOfMonth(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toLastDayNumOfMonth(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -242,7 +264,14 @@ struct ToStartOfQuarterImpl { return time_zone.toFirstDayNumOfQuarter(DayNum(d)); } - + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -266,6 +295,14 @@ struct ToStartOfYearImpl { return time_zone.toFirstDayNumOfYear(DayNum(d)); } + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfYear(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -893,7 +930,7 @@ struct ToStartOfISOYearImpl static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t)); + return t < 0 ? 0 : time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(std::min(Int32(time_zone.toDayNum(t)), Int32(DATE_LUT_MAX_DAY_NUM)))); } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { @@ -901,12 +938,20 @@ struct ToStartOfISOYearImpl } static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d)); + return d < 0 ? 0 : time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(std::min(d, Int32(DATE_LUT_MAX_DAY_NUM)))); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfISOYear(DayNum(d)); } + static inline Int64 execute_extended_result(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t)); + } + static inline Int32 execute_extended_result(Int32 d, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d)); + } using FactorTransform = ZeroTransform; }; @@ -1201,7 +1246,7 @@ struct ToYYYYMMDDhhmmssImpl }; -template +template struct Transformer { template @@ -1211,18 +1256,21 @@ struct Transformer vec_to.resize(size); for (size_t i = 0; i < size; ++i) - vec_to[i] = transform.execute(vec_from[i], time_zone); + if constexpr (is_extended_result) + vec_to[i] = transform.execute_extended_result(vec_from[i], time_zone); + else + vec_to[i] = transform.execute(vec_from[i], time_zone); } }; -template +template struct DateTimeTransformImpl { static ColumnPtr execute( const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/, const Transform & transform = {}) { - using Op = Transformer; + using Op = Transformer; const ColumnPtr source_col = arguments[0].column; if (const auto * sources = checkAndGetColumn(source_col.get())) diff --git a/src/Functions/FunctionCustomWeekToDateOrDate32.h b/src/Functions/FunctionCustomWeekToDateOrDate32.h new file mode 100644 index 00000000000..0b91fbb3bbe --- /dev/null +++ b/src/Functions/FunctionCustomWeekToDateOrDate32.h @@ -0,0 +1,78 @@ +#pragma once +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +template +class FunctionCustomWeekToDateOrDate32 : public IFunctionCustomWeek, WithContext +{ +public: + const bool enable_extended_results_for_datetime_functions = false; + + static FunctionPtr create(ContextPtr context_) + { + return std::make_shared(context_); + } + + explicit FunctionCustomWeekToDateOrDate32(ContextPtr context_) + : WithContext(context_) + , enable_extended_results_for_datetime_functions(context_->getSettingsRef().enable_extended_results_for_datetime_functions) + { + } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + this->checkArguments(arguments, /*is_result_type_date_or_date32*/ true); + + const IDataType * from_type = arguments[0].type.get(); + WhichDataType which(from_type); + if ((which.isDate32() || which.isDateTime64()) && enable_extended_results_for_datetime_functions) + return std::make_shared(); + else + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + const IDataType * from_type = arguments[0].type.get(); + WhichDataType which(from_type); + + if (which.isDate()) + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, Transform{}); + else if (which.isDate32()) + if (enable_extended_results_for_datetime_functions) + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, Transform{}); + else + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, Transform{}); + else if (which.isDateTime()) + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, Transform{}); + else if (which.isDateTime64()) + { + if (enable_extended_results_for_datetime_functions) + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, + TransformDateTime64{assert_cast(from_type)->getScale()}); + else + return CustomWeekTransformImpl::execute( + arguments, result_type, input_rows_count, + TransformDateTime64{assert_cast(from_type)->getScale()}); + } + else + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + this->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + +}; + +} diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index 8a0f474a7e8..eb65d562221 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -1,14 +1,5 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include - +#include namespace DB { @@ -16,82 +7,23 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } /// See CustomWeekTransforms.h template -class FunctionCustomWeekToSomething : public IFunction +class FunctionCustomWeekToSomething : public IFunctionCustomWeek { public: - static constexpr auto name = Transform::name; static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override { return name; } - - bool isVariadic() const override { return true; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() == 1) - { - if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception( - "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() - + ". Must be Date, Date32, DateTime or DateTime64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else if (arguments.size() == 2) - { - if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception( - "Illegal type " + arguments[0].type->getName() + " of 1st argument of function " + getName() - + ". Must be Date, Date32, DateTime or DateTime64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!isUInt8(arguments[1].type)) - throw Exception( - "Illegal type of 2nd (optional) argument of function " + getName() - + ". Must be constant UInt8 (week mode).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else if (arguments.size() == 3) - { - if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception( - "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() - + ". Must be Date, Date32, DateTime or DateTime64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!isUInt8(arguments[1].type)) - throw Exception( - "Illegal type of 2nd (optional) argument of function " + getName() - + ". Must be constant UInt8 (week mode).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!isString(arguments[2].type)) - throw Exception( - "Illegal type of 3rd (optional) argument of function " + getName() - + ". Must be constant string (timezone name).", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) - && (std::is_same_v || std::is_same_v)) - throw Exception( - "The timezone argument of function " + getName() + " is allowed only when the 1st argument is DateTime or DateTime64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else - throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) - + ", expected 1, 2 or 3.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + this->checkArguments(arguments); return std::make_shared(); } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const IDataType * from_type = arguments[0].type.get(); @@ -114,44 +46,10 @@ public: } else throw Exception( - "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(), + "Illegal type " + arguments[0].type->getName() + " of argument of function " + this->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - - bool hasInformationAboutMonotonicity() const override { return true; } - - Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override - { - if constexpr (std::is_same_v) - return { .is_monotonic = true, .is_always_monotonic = true }; - - const IFunction::Monotonicity is_monotonic = { .is_monotonic = true }; - const IFunction::Monotonicity is_not_monotonic; - - /// This method is called only if the function has one argument. Therefore, we do not care about the non-local time zone. - const DateLUTImpl & date_lut = DateLUT::instance(); - - if (left.isNull() || right.isNull()) - return {}; - - /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. - - if (checkAndGetDataType(&type)) - { - return Transform::FactorTransform::execute(UInt16(left.get()), date_lut) - == Transform::FactorTransform::execute(UInt16(right.get()), date_lut) - ? is_monotonic - : is_not_monotonic; - } - else - { - return Transform::FactorTransform::execute(UInt32(left.get()), date_lut) - == Transform::FactorTransform::execute(UInt32(right.get()), date_lut) - ? is_monotonic - : is_not_monotonic; - } - } }; } diff --git a/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h b/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h new file mode 100644 index 00000000000..3ff90cb57fb --- /dev/null +++ b/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h @@ -0,0 +1,81 @@ +#pragma once +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +template +class FunctionDateOrDateTimeToDateOrDate32 : public IFunctionDateOrDateTime, WithContext +{ +public: + const bool enable_extended_results_for_datetime_functions = false; + + static FunctionPtr create(ContextPtr context_) + { + return std::make_shared(context_); + } + + explicit FunctionDateOrDateTimeToDateOrDate32(ContextPtr context_) + : WithContext(context_) + , enable_extended_results_for_datetime_functions(context_->getSettingsRef().enable_extended_results_for_datetime_functions) + { + } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + this->checkArguments(arguments, /*is_result_type_date_or_date32*/ true); + + const IDataType * from_type = arguments[0].type.get(); + WhichDataType which(from_type); + + /// If the time zone is specified but empty, throw an exception. + /// only validate the time_zone part if the number of arguments is 2. + if ((which.isDateTime() || which.isDateTime64()) && arguments.size() == 2 + && extractTimeZoneNameFromFunctionArguments(arguments, 1, 0).empty()) + throw Exception( + "Function " + this->getName() + " supports a 2nd argument (optional) that must be non-empty and be a valid time zone", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if ((which.isDate32() || which.isDateTime64()) && enable_extended_results_for_datetime_functions) + return std::make_shared(); + else + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + const IDataType * from_type = arguments[0].type.get(); + WhichDataType which(from_type); + + if (which.isDate()) + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count); + else if (which.isDate32()) + if (enable_extended_results_for_datetime_functions) + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count); + else + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count); + else if (which.isDateTime()) + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count); + else if (which.isDateTime64()) + { + const auto scale = static_cast(from_type)->getScale(); + + const TransformDateTime64 transformer(scale); + if (enable_extended_results_for_datetime_functions) + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count, transformer); + else + return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count, transformer); + } + else + throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + this->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + +}; + +} diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index d734c7f87c1..5c1c54c1b84 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -1,14 +1,5 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include - +#include namespace DB { @@ -16,59 +7,18 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } /// See DateTimeTransforms.h template -class FunctionDateOrDateTimeToSomething : public IFunction +class FunctionDateOrDateTimeToSomething : public IFunctionDateOrDateTime { public: - static constexpr auto name = Transform::name; static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override - { - return name; - } - - bool isVariadic() const override { return true; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - size_t getNumberOfArguments() const override { return 0; } - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() == 1) - { - if (!isDateOrDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception( - "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() - + ". Should be a date or a date with time", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else if (arguments.size() == 2) - { - if (!isDateOrDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception( - "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() - + ". Should be a date or a date with time", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!isString(arguments[1].type)) - throw Exception( - "Function " + getName() + " supports 1 or 2 arguments. The 1st argument " - "must be of type Date or DateTime. The 2nd argument (optional) must be " - "a constant string with timezone name", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) && (std::is_same_v || std::is_same_v)) - throw Exception( - "The timezone argument of function " + getName() + " is allowed only when the 1st argument has the type DateTime", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else - throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) - + ", should be 1 or 2", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + this->checkArguments(arguments, (std::is_same_v || std::is_same_v)); /// For DateTime, if time zone is specified, attach it to type. /// If the time zone is specified but empty, throw an exception. @@ -79,7 +29,7 @@ public: /// to accommodate functions like toStartOfDay(today()), toStartOfDay(yesterday()) etc. if (arguments.size() == 2 && time_zone.empty()) throw Exception( - "Function " + getName() + " supports a 2nd argument (optional) that must be non-empty and be a valid time zone", + "Function " + this->getName() + " supports a 2nd argument (optional) that must be non-empty and be a valid time zone", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(time_zone); } @@ -109,9 +59,6 @@ public: return std::make_shared(); } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const IDataType * from_type = arguments[0].type.get(); @@ -131,51 +78,10 @@ public: return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count, transformer); } else - throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(), + throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + this->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - bool hasInformationAboutMonotonicity() const override - { - return true; - } - - Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override - { - if constexpr (std::is_same_v) - return { .is_monotonic = true, .is_always_monotonic = true }; - - const IFunction::Monotonicity is_monotonic = { .is_monotonic = true }; - const IFunction::Monotonicity is_not_monotonic; - - const DateLUTImpl * date_lut = &DateLUT::instance(); - if (const auto * timezone = dynamic_cast(&type)) - date_lut = &timezone->getTimeZone(); - - if (left.isNull() || right.isNull()) - return is_not_monotonic; - - /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. - - if (checkAndGetDataType(&type)) - { - return Transform::FactorTransform::execute(UInt16(left.get()), *date_lut) - == Transform::FactorTransform::execute(UInt16(right.get()), *date_lut) - ? is_monotonic : is_not_monotonic; - } - else if (checkAndGetDataType(&type)) - { - return Transform::FactorTransform::execute(Int32(left.get()), *date_lut) - == Transform::FactorTransform::execute(Int32(right.get()), *date_lut) - ? is_monotonic : is_not_monotonic; - } - else - { - return Transform::FactorTransform::execute(UInt32(left.get()), *date_lut) - == Transform::FactorTransform::execute(UInt32(right.get()), *date_lut) - ? is_monotonic : is_not_monotonic; - } - } }; } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 96c28b21ef0..a8d2882c653 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -2381,7 +2381,7 @@ using FunctionToDate = FunctionConvert; using FunctionToDateTime = FunctionConvert; using FunctionToDateTime32 = FunctionConvert; -using FunctionToDateTime64 = FunctionConvert; +using FunctionToDateTime64 = FunctionConvert; using FunctionToUUID = FunctionConvert>; using FunctionToString = FunctionConvert; using FunctionToUnixTimestamp = FunctionConvert>; diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index 20be3ee3cce..aa2144d271f 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -649,7 +649,7 @@ public: for (unsigned int region_id : region_ids) { const StringRef & name_ref = dict.getRegionName(region_id, language); - col_to->insertDataWithTerminatingZero(name_ref.data, name_ref.size + 1); + col_to->insertData(name_ref.data, name_ref.size); } return col_to; diff --git a/src/Functions/GatherUtils/Sources.h b/src/Functions/GatherUtils/Sources.h index 13e3de99552..b78e70975c6 100644 --- a/src/Functions/GatherUtils/Sources.h +++ b/src/Functions/GatherUtils/Sources.h @@ -140,17 +140,12 @@ struct NumericArraySource : public ArraySourceImpl> /// The methods can be virtual or not depending on the template parameter. See IStringSource. -#if !defined(__clang__) -# pragma GCC diagnostic push +#pragma GCC diagnostic push +#ifdef HAS_SUGGEST_OVERRIDE # pragma GCC diagnostic ignored "-Wsuggest-override" -#elif __clang_major__ >= 11 -# pragma GCC diagnostic push -# ifdef HAS_SUGGEST_OVERRIDE -# pragma GCC diagnostic ignored "-Wsuggest-override" -# endif -# ifdef HAS_SUGGEST_DESTRUCTOR_OVERRIDE -# pragma GCC diagnostic ignored "-Wsuggest-destructor-override" -# endif +#endif +#ifdef HAS_SUGGEST_DESTRUCTOR_OVERRIDE +# pragma GCC diagnostic ignored "-Wsuggest-destructor-override" #endif template @@ -233,9 +228,7 @@ struct ConstSource : public Base } }; -#if !defined(__clang__) || __clang_major__ >= 11 -# pragma GCC diagnostic pop -#endif +#pragma GCC diagnostic pop struct StringSource { diff --git a/src/Functions/IFunctionCustomWeek.h b/src/Functions/IFunctionCustomWeek.h new file mode 100644 index 00000000000..1bc4e44655a --- /dev/null +++ b/src/Functions/IFunctionCustomWeek.h @@ -0,0 +1,122 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +template +class IFunctionCustomWeek : public IFunction +{ +public: + static constexpr auto name = Transform::name; + String getName() const override { return name; } + bool isVariadic() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } + + bool hasInformationAboutMonotonicity() const override { return true; } + + Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override + { + if constexpr (std::is_same_v) + return {.is_monotonic = true, .is_always_monotonic = true}; + + const IFunction::Monotonicity is_monotonic = {.is_monotonic = true}; + const IFunction::Monotonicity is_not_monotonic; + + /// This method is called only if the function has one argument. Therefore, we do not care about the non-local time zone. + const DateLUTImpl & date_lut = DateLUT::instance(); + + if (left.isNull() || right.isNull()) + return {}; + + /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. + + if (checkAndGetDataType(&type)) + { + return Transform::FactorTransform::execute(UInt16(left.get()), date_lut) + == Transform::FactorTransform::execute(UInt16(right.get()), date_lut) + ? is_monotonic + : is_not_monotonic; + } + else + { + return Transform::FactorTransform::execute(UInt32(left.get()), date_lut) + == Transform::FactorTransform::execute(UInt32(right.get()), date_lut) + ? is_monotonic + : is_not_monotonic; + } + } + +protected: + void checkArguments(const ColumnsWithTypeAndName & arguments, bool is_result_type_date_or_date32 = false) const + { + if (arguments.size() == 1) + { + if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Must be Date, Date32, DateTime or DateTime64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else if (arguments.size() == 2) + { + if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of 1st argument of function " + getName() + + ". Must be Date, Date32, DateTime or DateTime64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isUInt8(arguments[1].type)) + throw Exception( + "Illegal type of 2nd (optional) argument of function " + getName() + + ". Must be constant UInt8 (week mode).", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else if (arguments.size() == 3) + { + if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Must be Date, Date32, DateTime or DateTime64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isUInt8(arguments[1].type)) + throw Exception( + "Illegal type of 2nd (optional) argument of function " + getName() + + ". Must be constant UInt8 (week mode).", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isString(arguments[2].type)) + throw Exception( + "Illegal type of 3rd (optional) argument of function " + getName() + + ". Must be constant string (timezone name).", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) && is_result_type_date_or_date32) + throw Exception( + "The timezone argument of function " + getName() + " is allowed only when the 1st argument is DateTime or DateTime64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", expected 1, 2 or 3.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + } + +}; + +} diff --git a/src/Functions/IFunctionDateOrDateTime.h b/src/Functions/IFunctionDateOrDateTime.h new file mode 100644 index 00000000000..1efe89c7fe9 --- /dev/null +++ b/src/Functions/IFunctionDateOrDateTime.h @@ -0,0 +1,118 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +template +class IFunctionDateOrDateTime : public IFunction +{ +public: + static constexpr auto name = Transform::name; + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + size_t getNumberOfArguments() const override { return 0; } + + bool useDefaultImplementationForConstants() const override { return true; } + + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + bool hasInformationAboutMonotonicity() const override + { + return true; + } + + Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override + { + if constexpr (std::is_same_v) + return { .is_monotonic = true, .is_always_monotonic = true }; + + const IFunction::Monotonicity is_monotonic = { .is_monotonic = true }; + const IFunction::Monotonicity is_not_monotonic; + + const DateLUTImpl * date_lut = &DateLUT::instance(); + if (const auto * timezone = dynamic_cast(&type)) + date_lut = &timezone->getTimeZone(); + + if (left.isNull() || right.isNull()) + return is_not_monotonic; + + /// The function is monotonous on the [left, right] segment, if the factor transformation returns the same values for them. + + if (checkAndGetDataType(&type)) + { + return Transform::FactorTransform::execute(UInt16(left.get()), *date_lut) + == Transform::FactorTransform::execute(UInt16(right.get()), *date_lut) + ? is_monotonic : is_not_monotonic; + } + else if (checkAndGetDataType(&type)) + { + return Transform::FactorTransform::execute(Int32(left.get()), *date_lut) + == Transform::FactorTransform::execute(Int32(right.get()), *date_lut) + ? is_monotonic : is_not_monotonic; + } + else + { + return Transform::FactorTransform::execute(UInt32(left.get()), *date_lut) + == Transform::FactorTransform::execute(UInt32(right.get()), *date_lut) + ? is_monotonic : is_not_monotonic; + } + } + +protected: + void checkArguments(const ColumnsWithTypeAndName & arguments, bool is_result_type_date_or_date32) const + { + if (arguments.size() == 1) + { + if (!isDateOrDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Should be Date, Date32, DateTime or DateTime64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else if (arguments.size() == 2) + { + if (!isDateOrDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + + ". Should be Date, Date32, DateTime or DateTime64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isString(arguments[1].type)) + throw Exception( + "Function " + getName() + " supports 1 or 2 arguments. The optional 2nd argument must be " + "a constant string with a timezone name", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) && is_result_type_date_or_date32) + throw Exception( + "The timezone argument of function " + getName() + " is allowed only when the 1st argument has the type DateTime or DateTime64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be 1 or 2", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + } +}; + +} diff --git a/src/Functions/LowerUpperUTF8Impl.h b/src/Functions/LowerUpperUTF8Impl.h index a7475870dab..3a050e2bd6f 100644 --- a/src/Functions/LowerUpperUTF8Impl.h +++ b/src/Functions/LowerUpperUTF8Impl.h @@ -2,6 +2,7 @@ #include #include #include +#include #ifdef __SSE2__ #include @@ -89,9 +90,11 @@ struct LowerUpperUTF8Impl ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) { + if (data.empty()) + return; res_data.resize(data.size()); res_offsets.assign(offsets); - array(data.data(), data.data() + data.size(), res_data.data()); + array(data.data(), data.data() + data.size(), offsets, res_data.data()); } static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &) @@ -164,8 +167,11 @@ private: static constexpr auto ascii_upper_bound = '\x7f'; static constexpr auto flip_case_mask = 'A' ^ 'a'; - static void array(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) + static void array(const UInt8 * src, const UInt8 * src_end, const ColumnString::Offsets & offsets, UInt8 * dst) { + auto offset_it = offsets.begin(); + const UInt8 * begin = src; + #ifdef __SSE2__ static constexpr auto bytes_sse = sizeof(__m128i); const auto * src_end_sse = src + (src_end - src) / bytes_sse * bytes_sse; @@ -213,10 +219,17 @@ private: else { /// UTF-8 - const auto * expected_end = src + bytes_sse; + + size_t offset_from_begin = src - begin; + while (offset_from_begin >= *offset_it) + ++offset_it; + /// Do not allow one row influence another (since row may have invalid sequence, and break the next) + const UInt8 * row_end = begin + *offset_it; + chassert(row_end >= src); + const UInt8 * expected_end = std::min(src + bytes_sse, row_end); while (src < expected_end) - toCase(src, src_end, dst); + toCase(src, expected_end, dst); /// adjust src_end_sse by pushing it forward or backward const auto diff = src - expected_end; @@ -229,10 +242,22 @@ private: } } } + + /// Find which offset src has now + while (offset_it != offsets.end() && static_cast(src - begin) >= *offset_it) + ++offset_it; #endif - /// handle remaining symbols + + /// handle remaining symbols, row by row (to avoid influence of bad UTF8 symbols from one row, to another) while (src < src_end) - toCase(src, src_end, dst); + { + const UInt8 * row_end = begin + *offset_it; + chassert(row_end >= src); + + while (src < row_end) + toCase(src, row_end, dst); + ++offset_it; + } } }; diff --git a/src/Functions/PolygonUtils.h b/src/Functions/PolygonUtils.h index de4bb2d48de..c16db1d1ff7 100644 --- a/src/Functions/PolygonUtils.h +++ b/src/Functions/PolygonUtils.h @@ -12,11 +12,6 @@ /// Warning in boost::geometry during template strategy substitution. #pragma GCC diagnostic push - -#if !defined(__clang__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - #pragma GCC diagnostic ignored "-Wunused-parameter" #include @@ -286,16 +281,9 @@ void PointInPolygonWithGrid::calcGridAttributes( const Point & min_corner = box.min_corner(); const Point & max_corner = box.max_corner(); -#pragma GCC diagnostic push -#if !defined(__clang__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - cell_width = (max_corner.x() - min_corner.x()) / grid_size; cell_height = (max_corner.y() - min_corner.y()) / grid_size; -#pragma GCC diagnostic pop - if (cell_width == 0 || cell_height == 0) { has_empty_bound = true; @@ -330,10 +318,6 @@ void PointInPolygonWithGrid::buildGrid() for (size_t row = 0; row < grid_size; ++row) { -#pragma GCC diagnostic push -#if !defined(__clang__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif CoordinateType y_min = min_corner.y() + row * cell_height; CoordinateType y_max = min_corner.y() + (row + 1) * cell_height; @@ -341,7 +325,6 @@ void PointInPolygonWithGrid::buildGrid() { CoordinateType x_min = min_corner.x() + col * cell_width; CoordinateType x_max = min_corner.x() + (col + 1) * cell_width; -#pragma GCC diagnostic pop Box cell_box(Point(x_min, y_min), Point(x_max, y_max)); MultiPolygon intersection; diff --git a/src/Functions/TransformDateTime64.h b/src/Functions/TransformDateTime64.h index 9ac28118b8f..fbe7e2e8250 100644 --- a/src/Functions/TransformDateTime64.h +++ b/src/Functions/TransformDateTime64.h @@ -87,6 +87,46 @@ public: return wrapped_transform.execute(t, std::forward(args)...); } + + template + inline auto NO_SANITIZE_UNDEFINED execute_extended_result(const DateTime64 & t, Args && ... args) const + { + /// Type conversion from float to integer may be required. + /// We are Ok with implementation specific result for out of range and denormals conversion. + + if constexpr (TransformHasExecuteOverload_v) + { + return wrapped_transform.execute_extended_result(t, scale_multiplier, std::forward(args)...); + } + else if constexpr (TransformHasExecuteOverload_v, Args...>) + { + auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + + const auto result = wrapped_transform.execute_extended_result(components, std::forward(args)...); + using ResultType = std::decay_t; + + if constexpr (std::is_same_v, ResultType>) + { + return DecimalUtils::decimalFromComponentsWithMultiplier(result, scale_multiplier); + } + else + { + return result; + } + } + else + { + const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + return wrapped_transform.execute_extended_result(static_cast(components.whole), std::forward(args)...); + } + } + + template >> + inline auto execute_extended_result(const T & t, Args && ... args) const + { + return wrapped_transform.execute_extended_result(t, std::forward(args)...); + } + private: DateTime64::NativeType scale_multiplier = 1; Transform wrapped_transform = {}; diff --git a/src/Functions/addressToSymbol.cpp b/src/Functions/addressToSymbol.cpp index 99988ee82f6..dd9efd6cc44 100644 --- a/src/Functions/addressToSymbol.cpp +++ b/src/Functions/addressToSymbol.cpp @@ -83,7 +83,7 @@ public: for (size_t i = 0; i < input_rows_count; ++i) { if (const auto * symbol = symbol_index.findSymbol(reinterpret_cast(data[i]))) - result_column->insertDataWithTerminatingZero(symbol->name, strlen(symbol->name) + 1); + result_column->insertData(symbol->name, strlen(symbol->name)); else result_column->insertDefault(); } diff --git a/src/Functions/modelEvaluate.cpp b/src/Functions/catboostEvaluate.cpp similarity index 55% rename from src/Functions/modelEvaluate.cpp rename to src/Functions/catboostEvaluate.cpp index 3ee2ae3fae4..1ac7815239e 100644 --- a/src/Functions/modelEvaluate.cpp +++ b/src/Functions/catboostEvaluate.cpp @@ -1,18 +1,18 @@ #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include #include +#include #include -#include +#include #include +#include +#include +#include #include +#include #include @@ -21,66 +21,80 @@ namespace DB namespace ErrorCodes { + extern const int FILE_DOESNT_EXIST; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int ILLEGAL_COLUMN; } -class ExternalModelsLoader; - - -/// Evaluate external model. -/// First argument - model name, the others - model arguments. -/// * for CatBoost model - float features first, then categorical -/// Result - Float64. -class FunctionModelEvaluate final : public IFunction +/// Evaluate CatBoost model. +/// - Arguments: float features first, then categorical features. +/// - Result: Float64. +class FunctionCatBoostEvaluate final : public IFunction, WithContext { +private: + mutable std::unique_ptr bridge_helper; + public: - static constexpr auto name = "modelEvaluate"; + static constexpr auto name = "catboostEvaluate"; - static FunctionPtr create(ContextPtr context) - { - return std::make_shared(context->getExternalModelsLoader()); - } - - explicit FunctionModelEvaluate(const ExternalModelsLoader & models_loader_) - : models_loader(models_loader_) {} + static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + explicit FunctionCatBoostEvaluate(ContextPtr context_) : WithContext(context_) {} String getName() const override { return name; } - bool isVariadic() const override { return true; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - bool isDeterministic() const override { return false; } - bool useDefaultImplementationForNulls() const override { return false; } - size_t getNumberOfArguments() const override { return 0; } + void initBridge(const ColumnConst * name_col) const + { + String library_path = getContext()->getConfigRef().getString("catboost_lib_path"); + if (!std::filesystem::exists(library_path)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Can't load library {}: file doesn't exist", library_path); + + String model_path = name_col->getValue(); + if (!std::filesystem::exists(model_path)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Can't load model {}: file doesn't exist", model_path); + + bridge_helper = std::make_unique(getContext(), model_path, library_path); + } + + DataTypePtr getReturnTypeFromLibraryBridge() const + { + size_t tree_count = bridge_helper->getTreeCount(); + auto type = std::make_shared(); + if (tree_count == 1) + return type; + + DataTypes types(tree_count, type); + + return std::make_shared(types); + } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { if (arguments.size() < 2) - throw Exception("Function " + getName() + " expects at least 2 arguments", - ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); + throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Function {} expects at least 2 arguments", getName()); if (!isString(arguments[0].type)) - throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() - + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of first argument of function {}, expected a string.", arguments[0].type->getName(), getName()); const auto * name_col = checkAndGetColumnConst(arguments[0].column.get()); if (!name_col) - throw Exception("First argument of function " + getName() + " must be a constant string", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "First argument of function {} must be a constant string", getName()); + + initBridge(name_col); + + auto type = getReturnTypeFromLibraryBridge(); bool has_nullable = false; for (size_t i = 1; i < arguments.size(); ++i) has_nullable = has_nullable || arguments[i].type->isNullable(); - auto model = models_loader.getModel(name_col->getValue()); - auto type = model->getReturnType(); - if (has_nullable) { if (const auto * tuple = typeid_cast(type.get())) @@ -98,31 +112,25 @@ public: return type; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override { const auto * name_col = checkAndGetColumnConst(arguments[0].column.get()); if (!name_col) - throw Exception("First argument of function " + getName() + " must be a constant string", - ErrorCodes::ILLEGAL_COLUMN); - - auto model = models_loader.getModel(name_col->getValue()); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "First argument of function {} must be a constant string", getName()); ColumnRawPtrs column_ptrs; Columns materialized_columns; ColumnPtr null_map; - column_ptrs.reserve(arguments.size()); - for (auto arg : collections::range(1, arguments.size())) + ColumnsWithTypeAndName feature_arguments(arguments.begin() + 1, arguments.end()); + for (auto & arg : feature_arguments) { - const auto & column = arguments[arg].column; - column_ptrs.push_back(column.get()); - if (auto full_column = column->convertToFullColumnIfConst()) + if (auto full_column = arg.column->convertToFullColumnIfConst()) { materialized_columns.push_back(full_column); - column_ptrs.back() = full_column.get(); + arg.column = full_column; } - if (const auto * col_nullable = checkAndGetColumn(*column_ptrs.back())) + if (const auto * col_nullable = checkAndGetColumn(&*arg.column)) { if (!null_map) null_map = col_nullable->getNullMapColumnPtr(); @@ -140,11 +148,12 @@ public: null_map = std::move(mut_null_map); } - column_ptrs.back() = &col_nullable->getNestedColumn(); + arg.column = col_nullable->getNestedColumn().getPtr(); + arg.type = static_cast(*arg.type).getNestedType(); } } - auto res = model->evaluate(column_ptrs); + auto res = bridge_helper->evaluate(feature_arguments); if (null_map) { @@ -162,15 +171,12 @@ public: return res; } - -private: - const ExternalModelsLoader & models_loader; }; -REGISTER_FUNCTION(ExternalModels) +REGISTER_FUNCTION(CatBoostEvaluate) { - factory.registerFunction(); + factory.registerFunction(); } } diff --git a/src/Functions/demange.cpp b/src/Functions/demange.cpp index 9026790f740..a7c3d8e52bf 100644 --- a/src/Functions/demange.cpp +++ b/src/Functions/demange.cpp @@ -78,15 +78,15 @@ public: for (size_t i = 0; i < input_rows_count; ++i) { - StringRef source = column_concrete->getDataAtWithTerminatingZero(i); + StringRef source = column_concrete->getDataAt(i); auto demangled = tryDemangle(source.data); if (demangled) { - result_column->insertDataWithTerminatingZero(demangled.get(), strlen(demangled.get()) + 1); + result_column->insertData(demangled.get(), strlen(demangled.get())); } else { - result_column->insertDataWithTerminatingZero(source.data, source.size); + result_column->insertData(source.data, source.size); } } @@ -102,4 +102,3 @@ REGISTER_FUNCTION(Demangle) } } - diff --git a/src/Functions/grouping.h b/src/Functions/grouping.h index a49e946b2cb..b9ef6ffc107 100644 --- a/src/Functions/grouping.h +++ b/src/Functions/grouping.h @@ -1,9 +1,9 @@ #pragma once +#include #include #include #include -#include #include #include #include @@ -19,10 +19,17 @@ protected: static constexpr UInt64 ONE = 1; const ColumnNumbers arguments_indexes; + // Initial implementation of GROUPING function returned 1 if the argument is used as an aggregation key. + // This differs from the behavior described in the standard and other DBMS. + const bool force_compatibility; + + static constexpr UInt64 COMPATIBLE_MODE[] = {1, 0}; + static constexpr UInt64 INCOMPATIBLE_MODE[] = {0, 1}; public: - FunctionGroupingBase(ColumnNumbers arguments_indexes_) + FunctionGroupingBase(ColumnNumbers arguments_indexes_, bool force_compatibility_) : arguments_indexes(std::move(arguments_indexes_)) + , force_compatibility(force_compatibility_) {} bool isVariadic() const override { return true; } @@ -48,13 +55,15 @@ public: auto result = ColumnUInt64::create(); auto & result_data = result->getData(); result_data.reserve(input_rows_count); + + const auto * result_table = likely(force_compatibility) ? COMPATIBLE_MODE : INCOMPATIBLE_MODE; for (size_t i = 0; i < input_rows_count; ++i) { UInt64 set_index = grouping_set_column->getElement(i); UInt64 value = 0; for (auto index : arguments_indexes) - value = (value << 1) + (checker(set_index, index) ? 1 : 0); + value = (value << 1) + result_table[checker(set_index, index) ? 1 : 0]; result_data.push_back(value); } @@ -65,14 +74,16 @@ public: class FunctionGroupingOrdinary : public FunctionGroupingBase { public: - explicit FunctionGroupingOrdinary(ColumnNumbers arguments_indexes_) - : FunctionGroupingBase(std::move(arguments_indexes_)) + FunctionGroupingOrdinary(ColumnNumbers arguments_indexes_, bool force_compatibility_) + : FunctionGroupingBase(std::move(arguments_indexes_), force_compatibility_) {} String getName() const override { return "groupingOrdinary"; } ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr &, size_t input_rows_count) const override { + if (likely(force_compatibility)) + return ColumnUInt64::create(input_rows_count, 0); UInt64 value = (ONE << arguments_indexes.size()) - 1; return ColumnUInt64::create(input_rows_count, value); } @@ -83,8 +94,8 @@ class FunctionGroupingForRollup : public FunctionGroupingBase const UInt64 aggregation_keys_number; public: - FunctionGroupingForRollup(ColumnNumbers arguments_indexes_, UInt64 aggregation_keys_number_) - : FunctionGroupingBase(std::move(arguments_indexes_)) + FunctionGroupingForRollup(ColumnNumbers arguments_indexes_, UInt64 aggregation_keys_number_, bool force_compatibility_) + : FunctionGroupingBase(std::move(arguments_indexes_), force_compatibility_) , aggregation_keys_number(aggregation_keys_number_) {} @@ -113,8 +124,8 @@ class FunctionGroupingForCube : public FunctionGroupingBase public: - FunctionGroupingForCube(ColumnNumbers arguments_indexes_, UInt64 aggregation_keys_number_) - : FunctionGroupingBase(arguments_indexes_) + FunctionGroupingForCube(ColumnNumbers arguments_indexes_, UInt64 aggregation_keys_number_, bool force_compatibility_) + : FunctionGroupingBase(arguments_indexes_, force_compatibility_) , aggregation_keys_number(aggregation_keys_number_) {} @@ -142,8 +153,8 @@ class FunctionGroupingForGroupingSets : public FunctionGroupingBase { ColumnNumbersSetList grouping_sets; public: - FunctionGroupingForGroupingSets(ColumnNumbers arguments_indexes_, ColumnNumbersList const & grouping_sets_) - : FunctionGroupingBase(std::move(arguments_indexes_)) + FunctionGroupingForGroupingSets(ColumnNumbers arguments_indexes_, ColumnNumbersList const & grouping_sets_, bool force_compatibility_) + : FunctionGroupingBase(std::move(arguments_indexes_), force_compatibility_) { for (auto const & set : grouping_sets_) grouping_sets.emplace_back(set.begin(), set.end()); diff --git a/src/Functions/toCustomWeek.cpp b/src/Functions/toCustomWeek.cpp index 13dc76b6389..b773cc7df96 100644 --- a/src/Functions/toCustomWeek.cpp +++ b/src/Functions/toCustomWeek.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -9,7 +10,7 @@ namespace DB { using FunctionToWeek = FunctionCustomWeekToSomething; using FunctionToYearWeek = FunctionCustomWeekToSomething; -using FunctionToStartOfWeek = FunctionCustomWeekToSomething; +using FunctionToStartOfWeek = FunctionCustomWeekToDateOrDate32; REGISTER_FUNCTION(ToCustomWeek) { diff --git a/src/Functions/toLastDayOfMonth.cpp b/src/Functions/toLastDayOfMonth.cpp index a7faab15f9f..9365880bfb8 100644 --- a/src/Functions/toLastDayOfMonth.cpp +++ b/src/Functions/toLastDayOfMonth.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToLastDayOfMonth = FunctionDateOrDateTimeToSomething; +using FunctionToLastDayOfMonth = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToLastDayOfMonth) { diff --git a/src/Functions/toMonday.cpp b/src/Functions/toMonday.cpp index 89145634e45..280c8a93b1a 100644 --- a/src/Functions/toMonday.cpp +++ b/src/Functions/toMonday.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToMonday = FunctionDateOrDateTimeToSomething; +using FunctionToMonday = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToMonday) { diff --git a/src/Functions/toStartOfISOYear.cpp b/src/Functions/toStartOfISOYear.cpp index 366ba8dd09f..245d043f0b6 100644 --- a/src/Functions/toStartOfISOYear.cpp +++ b/src/Functions/toStartOfISOYear.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToStartOfISOYear = FunctionDateOrDateTimeToSomething; +using FunctionToStartOfISOYear = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToStartOfISOYear) { diff --git a/src/Functions/toStartOfMonth.cpp b/src/Functions/toStartOfMonth.cpp index 9674462097b..00146e25d44 100644 --- a/src/Functions/toStartOfMonth.cpp +++ b/src/Functions/toStartOfMonth.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToStartOfMonth = FunctionDateOrDateTimeToSomething; +using FunctionToStartOfMonth = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToStartOfMonth) { diff --git a/src/Functions/toStartOfQuarter.cpp b/src/Functions/toStartOfQuarter.cpp index c7d69743198..74966d51584 100644 --- a/src/Functions/toStartOfQuarter.cpp +++ b/src/Functions/toStartOfQuarter.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToStartOfQuarter = FunctionDateOrDateTimeToSomething; +using FunctionToStartOfQuarter = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToStartOfQuarter) { diff --git a/src/Functions/toStartOfYear.cpp b/src/Functions/toStartOfYear.cpp index 13729f2f812..27019bfd69f 100644 --- a/src/Functions/toStartOfYear.cpp +++ b/src/Functions/toStartOfYear.cpp @@ -1,12 +1,12 @@ #include #include -#include +#include namespace DB { -using FunctionToStartOfYear = FunctionDateOrDateTimeToSomething; +using FunctionToStartOfYear = FunctionDateOrDateTimeToDateOrDate32; REGISTER_FUNCTION(ToStartOfYear) { diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp index 3337e8d40a8..62be535be85 100644 --- a/src/Functions/transform.cpp +++ b/src/Functions/transform.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -920,8 +921,7 @@ private: ColumnString::Offset current_dst_default_offset = 0; for (size_t i = 0; i < size; ++i) { - Field key = src[i]; - const auto * it = table.find(key.reinterpret()); + const auto * it = table.find(bit_cast(src[i])); StringRef ref; if (it) @@ -1081,6 +1081,22 @@ private: mutable Cache cache; + + static UInt64 bitCastToUInt64(const Field & x) + { + switch (x.getType()) + { + case Field::Types::UInt64: return x.get(); + case Field::Types::Int64: return x.get(); + case Field::Types::Float64: return std::bit_cast(x.get()); + case Field::Types::Bool: return x.get(); + case Field::Types::Decimal32: return x.get>().getValue(); + case Field::Types::Decimal64: return x.get>().getValue(); + default: + throw Exception("Unexpected type in function 'transform'", ErrorCodes::BAD_ARGUMENTS); + } + } + /// Can be called from different threads. It works only on the first call. void initialize(const Array & from, const Array & to, const ColumnsWithTypeAndName & arguments) const { @@ -1151,20 +1167,8 @@ private: if (key.isNull()) continue; - // Field may be of Float type, but for the purpose of bitwise - // equality we can treat them as UInt64, hence the reinterpret(). - if (to[0].getType() ==Field::Types::Decimal32) - { - table[key.reinterpret()] = (*used_to)[i].reinterpret(); - } - else if (to[0].getType() ==Field::Types::Decimal64) - { - table[key.reinterpret()] = (*used_to)[i].reinterpret(); - } - else - { - table[key.reinterpret()] = (*used_to)[i].reinterpret(); - } + /// Field may be of Float type, but for the purpose of bitwise equality we can treat them as UInt64 + table[bitCastToUInt64(key)] = bitCastToUInt64((*used_to)[i]); } } else @@ -1179,7 +1183,7 @@ private: const String & str_to = to[i].get(); StringRef ref{cache.string_pool.insert(str_to.data(), str_to.size() + 1), str_to.size() + 1}; - table[key.reinterpret()] = ref; + table[bitCastToUInt64(key)] = ref; } } } @@ -1193,7 +1197,7 @@ private: { const String & str_from = from[i].get(); StringRef ref{cache.string_pool.insert(str_from.data(), str_from.size() + 1), str_from.size() + 1}; - table[ref] = (*used_to)[i].reinterpret(); + table[ref] = bitCastToUInt64((*used_to)[i]); } } else diff --git a/src/Functions/trap.cpp b/src/Functions/trap.cpp index 5bb0ed210de..16fada0f67d 100644 --- a/src/Functions/trap.cpp +++ b/src/Functions/trap.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace DB @@ -25,6 +26,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; extern const int CANNOT_ALLOCATE_MEMORY; + extern const int CANNOT_DLOPEN; } @@ -136,7 +138,7 @@ public: } else if (mode == "access context") { - (void)context.getCurrentQueryId(); + (void)context->getCurrentQueryId(); } else if (mode == "stack overflow") { @@ -166,6 +168,12 @@ public: maps.push_back(map); } } + else if (mode == "dlopen") + { + void * handle = dlopen("libc.so.6", RTLD_NOW); + if (!handle) + throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror()); // NOLINT(concurrency-mt-unsafe) // MT-Safe on Linux, see man dlerror + } else throw Exception("Unknown trap mode", ErrorCodes::BAD_ARGUMENTS); } diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp index ffb7bff8afb..cb4b6ca5f3e 100644 --- a/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/src/IO/ReadBufferFromFileDescriptor.cpp @@ -233,7 +233,7 @@ void ReadBufferFromFileDescriptor::rewind() /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout. -bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds) +bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds) const { fd_set fds; FD_ZERO(&fds); diff --git a/src/IO/ReadBufferFromFileDescriptor.h b/src/IO/ReadBufferFromFileDescriptor.h index 6b68b8b6dfd..6edda460bac 100644 --- a/src/IO/ReadBufferFromFileDescriptor.h +++ b/src/IO/ReadBufferFromFileDescriptor.h @@ -66,7 +66,7 @@ public: private: /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout. - bool poll(size_t timeout_microseconds); + bool poll(size_t timeout_microseconds) const; }; diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 380365f9b95..c17bf731c62 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -24,6 +24,8 @@ namespace ProfileEvents extern const Event ReadBufferFromS3Bytes; extern const Event ReadBufferFromS3RequestsErrors; extern const Event ReadBufferSeekCancelConnection; + extern const Event S3GetObject; + extern const Event DiskS3GetObject; } namespace DB @@ -131,18 +133,18 @@ bool ReadBufferFromS3::nextImpl() ProfileEvents::increment(ProfileEvents::ReadBufferFromS3Microseconds, watch.elapsedMicroseconds()); break; } - catch (const Exception & e) + catch (Exception & e) { watch.stop(); ProfileEvents::increment(ProfileEvents::ReadBufferFromS3Microseconds, watch.elapsedMicroseconds()); ProfileEvents::increment(ProfileEvents::ReadBufferFromS3RequestsErrors, 1); - if (const auto * s3_exception = dynamic_cast(&e)) + if (auto * s3_exception = dynamic_cast(&e)) { /// It doesn't make sense to retry Access Denied or No Such Key if (!s3_exception->isRetryableError()) { - tryLogCurrentException(log, fmt::format("while reading key: {}, from bucket: {}", key, bucket)); + s3_exception->addMessage("while reading key: {}, from bucket: {}", key, bucket); throw; } } @@ -248,7 +250,7 @@ size_t ReadBufferFromS3::getFileSize() if (file_size) return *file_size; - auto object_size = S3::getObjectSize(client_ptr, bucket, key, version_id); + auto object_size = S3::getObjectSize(client_ptr, bucket, key, version_id, true, read_settings.for_object_storage); file_size = object_size; return *file_size; @@ -315,6 +317,10 @@ std::unique_ptr ReadBufferFromS3::initialize() offset); } + ProfileEvents::increment(ProfileEvents::S3GetObject); + if (read_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3GetObject); + Aws::S3::Model::GetObjectOutcome outcome = client_ptr->GetObject(req); if (outcome.IsSuccess()) diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index fb093c52f67..1fa4aa637f5 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -76,6 +76,8 @@ struct ReadSettings /// For 'pread_threadpool' method. Lower is more priority. size_t priority = 0; + bool load_marks_asynchronously = true; + size_t remote_fs_read_max_backoff_ms = 10000; size_t remote_fs_read_backoff_max_tries = 4; @@ -103,6 +105,9 @@ struct ReadSettings size_t http_retry_max_backoff_ms = 1600; bool http_skip_not_found_url_for_globs = true; + /// Monitoring + bool for_object_storage = false; // to choose which profile events should be incremented + ReadSettings adjustBufferSize(size_t file_size) const { ReadSettings res = *this; diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 63496054e85..60885da6da3 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -26,6 +26,7 @@ #include #include +#include namespace ProfileEvents { @@ -346,13 +347,29 @@ namespace detail non_retriable_errors.begin(), non_retriable_errors.end(), [&](const auto status) { return http_status != status; }); } + Poco::URI getUriAfterRedirect(const Poco::URI & prev_uri, Poco::Net::HTTPResponse & response) + { + auto location = response.get("Location"); + auto location_uri = Poco::URI(location); + if (!location_uri.isRelative()) + return location_uri; + /// Location header contains relative path. So we need to concatenate it + /// with path from the original URI and normalize it. + auto path = std::filesystem::weakly_canonical(std::filesystem::path(prev_uri.getPath()) / location); + location_uri = prev_uri; + location_uri.setPath(path); + return location_uri; + } + void callWithRedirects(Poco::Net::HTTPResponse & response, const String & method_, bool throw_on_all_errors = false) { call(response, method_, throw_on_all_errors); + Poco::URI prev_uri = uri; while (isRedirect(response.getStatus())) { - Poco::URI uri_redirect(response.get("Location")); + Poco::URI uri_redirect = getUriAfterRedirect(prev_uri, response); + prev_uri = uri_redirect; if (remote_host_filter) remote_host_filter->checkURL(uri_redirect); @@ -408,7 +425,7 @@ namespace detail while (isRedirect(response.getStatus())) { - Poco::URI uri_redirect(response.get("Location")); + Poco::URI uri_redirect = getUriAfterRedirect(saved_uri_redirect.value_or(uri), response); if (remote_host_filter) remote_host_filter->checkURL(uri_redirect); diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index e97fa707c13..f807c8bc9b6 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -35,6 +35,12 @@ # include +namespace ProfileEvents +{ + extern const Event S3HeadObject; + extern const Event DiskS3HeadObject; +} + namespace DB { @@ -812,8 +818,12 @@ namespace S3 } - S3::ObjectInfo getObjectInfo(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error) + S3::ObjectInfo getObjectInfo(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3) { + ProfileEvents::increment(ProfileEvents::S3HeadObject); + if (for_disk_s3) + ProfileEvents::increment(ProfileEvents::DiskS3HeadObject); + Aws::S3::Model::HeadObjectRequest req; req.SetBucket(bucket); req.SetKey(key); @@ -835,9 +845,9 @@ namespace S3 return {}; } - size_t getObjectSize(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error) + size_t getObjectSize(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3) { - return getObjectInfo(client_ptr, bucket, key, version_id, throw_on_error).size; + return getObjectInfo(client_ptr, bucket, key, version_id, throw_on_error, for_disk_s3).size; } } diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index da7ecf95b78..05391b58403 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -55,7 +55,7 @@ public: bool isRetryableError() const; private: - const Aws::S3::S3Errors code; + Aws::S3::S3Errors code; }; } @@ -124,9 +124,9 @@ struct ObjectInfo time_t last_modification_time = 0; }; -S3::ObjectInfo getObjectInfo(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id = {}, bool throw_on_error = true); +S3::ObjectInfo getObjectInfo(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3); -size_t getObjectSize(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id = {}, bool throw_on_error = true); +size_t getObjectSize(std::shared_ptr client_ptr, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3); } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index b09abda85db..0215376aa0e 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -1,4 +1,5 @@ #include +#include #if USE_AWS_S3 @@ -24,6 +25,19 @@ namespace ProfileEvents { extern const Event WriteBufferFromS3Bytes; + extern const Event S3WriteBytes; + + extern const Event S3HeadObject; + extern const Event S3CreateMultipartUpload; + extern const Event S3CompleteMultipartUpload; + extern const Event S3UploadPart; + extern const Event S3PutObject; + + extern const Event DiskS3HeadObject; + extern const Event DiskS3CreateMultipartUpload; + extern const Event DiskS3CompleteMultipartUpload; + extern const Event DiskS3UploadPart; + extern const Event DiskS3PutObject; } namespace DB @@ -167,10 +181,15 @@ void WriteBufferFromS3::finalizeImpl() { LOG_TRACE(log, "Checking object {} exists after upload", key); + Aws::S3::Model::HeadObjectRequest request; request.SetBucket(bucket); request.SetKey(key); + ProfileEvents::increment(ProfileEvents::S3HeadObject); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3HeadObject); + auto response = client_ptr->HeadObject(request); if (!response.IsSuccess()) @@ -192,6 +211,10 @@ void WriteBufferFromS3::createMultipartUpload() if (object_metadata.has_value()) req.SetMetadata(object_metadata.value()); + ProfileEvents::increment(ProfileEvents::S3CreateMultipartUpload); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3CreateMultipartUpload); + auto outcome = client_ptr->CreateMultipartUpload(req); if (outcome.IsSuccess()) @@ -303,6 +326,10 @@ void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & re void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) { + ProfileEvents::increment(ProfileEvents::S3UploadPart); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3UploadPart); + auto outcome = client_ptr->UploadPart(task.req); if (outcome.IsSuccess()) @@ -340,16 +367,33 @@ void WriteBufferFromS3::completeMultipartUpload() req.SetMultipartUpload(multipart_upload); - auto outcome = client_ptr->CompleteMultipartUpload(req); - - if (outcome.IsSuccess()) - LOG_TRACE(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, tags.size()); - else + size_t max_retry = std::max(s3_settings.max_unexpected_write_error_retries, 1UL); + for (size_t i = 0; i < max_retry; ++i) { - throw S3Exception( - outcome.GetError().GetErrorType(), - "Message: {}, Key: {}, Bucket: {}, Tags: {}", - outcome.GetError().GetMessage(), key, bucket, fmt::join(tags.begin(), tags.end(), " ")); + ProfileEvents::increment(ProfileEvents::S3CompleteMultipartUpload); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); + + auto outcome = client_ptr->CompleteMultipartUpload(req); + + if (outcome.IsSuccess()) + { + LOG_TRACE(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, tags.size()); + break; + } + else if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) + { + /// For unknown reason, at least MinIO can respond with NO_SUCH_KEY for put requests + /// BTW, NO_SUCH_UPLOAD is expected error and we shouldn't retry it + LOG_INFO(log, "Multipart upload failed with NO_SUCH_KEY error for Bucket: {}, Key: {}, Upload_id: {}, Parts: {}, will retry", bucket, key, multipart_upload_id, tags.size()); + } + else + { + throw S3Exception( + outcome.GetError().GetErrorType(), + "Message: {}, Key: {}, Bucket: {}, Tags: {}", + outcome.GetError().GetMessage(), key, bucket, fmt::join(tags.begin(), tags.end(), " ")); + } } } @@ -429,15 +473,30 @@ void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req) void WriteBufferFromS3::processPutRequest(const PutObjectTask & task) { - auto outcome = client_ptr->PutObject(task.req); - bool with_pool = static_cast(schedule); - if (outcome.IsSuccess()) - LOG_TRACE(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); - else - throw S3Exception( - outcome.GetError().GetErrorType(), - "Message: {}, Key: {}, Bucket: {}, Object size: {}, WithPool: {}", - outcome.GetError().GetMessage(), key, bucket, task.req.GetContentLength(), with_pool); + size_t max_retry = std::max(s3_settings.max_unexpected_write_error_retries, 1UL); + for (size_t i = 0; i < max_retry; ++i) + { + ProfileEvents::increment(ProfileEvents::S3PutObject); + if (write_settings.for_object_storage) + ProfileEvents::increment(ProfileEvents::DiskS3PutObject); + auto outcome = client_ptr->PutObject(task.req); + bool with_pool = static_cast(schedule); + if (outcome.IsSuccess()) + { + LOG_TRACE(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); + break; + } + else if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) + { + /// For unknown reason, at least MinIO can respond with NO_SUCH_KEY for put requests + LOG_INFO(log, "Single part upload failed with NO_SUCH_KEY error for Bucket: {}, Key: {}, Object size: {}, WithPool: {}, will retry", bucket, key, task.req.GetContentLength(), with_pool); + } + else + throw S3Exception( + outcome.GetError().GetErrorType(), + "Message: {}, Key: {}, Bucket: {}, Object size: {}, WithPool: {}", + outcome.GetError().GetMessage(), key, bucket, task.req.GetContentLength(), with_pool); + } } void WriteBufferFromS3::waitForReadyBackGroundTasks() diff --git a/src/IO/WriteSettings.h b/src/IO/WriteSettings.h index ec2b5d17ec2..38a706997cf 100644 --- a/src/IO/WriteSettings.h +++ b/src/IO/WriteSettings.h @@ -15,6 +15,9 @@ struct WriteSettings bool enable_filesystem_cache_on_write_operations = false; bool enable_filesystem_cache_log = false; bool is_file_cache_persistent = false; + + /// Monitoring + bool for_object_storage = false; // to choose which profile events should be incremented }; } diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index df3f02708a9..cc0cf2576a6 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -45,12 +45,6 @@ inline size_t readAlpha(char * res, size_t max_chars, ReadBuffer & in) return num_chars; } -#if defined(__PPC__) -#if !defined(__clang__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif -#endif - template inline void readDecimalNumberImpl(T & res, const char * src) { @@ -656,12 +650,6 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf } -#if defined(__PPC__) -#if !defined(__clang__) -#pragma GCC diagnostic pop -#endif -#endif - void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index f10510fadae..54faf37f236 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -1,8 +1,9 @@ #include + #include #include -#include -#include +#include + #include #include @@ -22,9 +23,12 @@ #include #include #include +#include -#include +#include #include +#include +#include #include @@ -87,6 +91,22 @@ static size_t getTypeDepth(const DataTypePtr & type) return 0; } +/// Applies stricter rules than convertFieldToType: +/// Doesn't allow : +/// - loss of precision with `Decimals` +static bool convertFieldToTypeStrict(const Field & from_value, const IDataType & to_type, Field & result_value) +{ + result_value = convertFieldToType(from_value, to_type); + if (Field::isDecimal(from_value.getType()) && Field::isDecimal(result_value.getType())) + return applyVisitor(FieldVisitorAccurateEquals{}, from_value, result_value); + + return true; +} + +/// The `convertFieldToTypeStrict` is used to prevent unexpected results in case of conversion with loss of precision. +/// Example: `SELECT 33.3 :: Decimal(9, 1) AS a WHERE a IN (33.33 :: Decimal(9, 2))` +/// 33.33 in the set is converted to 33.3, but it is not equal to 33.3 in the column, so the result should still be empty. +/// We can not include values that don't represent any possible value from the type of filtered column to the set. template static Block createBlockFromCollection(const Collection & collection, const DataTypes & types, bool transform_null_in) { @@ -103,10 +123,11 @@ static Block createBlockFromCollection(const Collection & collection, const Data { if (columns_num == 1) { - auto field = convertFieldToType(value, *types[0]); + Field field; + bool is_conversion_ok = convertFieldToTypeStrict(value, *types[0], field); bool need_insert_null = transform_null_in && types[0]->isNullable(); - if (!field.isNull() || need_insert_null) - columns[0]->insert(std::move(field)); + if (is_conversion_ok && (!field.isNull() || need_insert_null)) + columns[0]->insert(field); } else { @@ -127,7 +148,10 @@ static Block createBlockFromCollection(const Collection & collection, const Data size_t i = 0; for (; i < tuple_size; ++i) { - tuple_values[i] = convertFieldToType(tuple[i], *types[i]); + bool is_conversion_ok = convertFieldToTypeStrict(tuple[i], *types[i], tuple_values[i]); + if (!is_conversion_ok) + break; + bool need_insert_null = transform_null_in && types[i]->isNullable(); if (tuple_values[i].isNull() && !need_insert_null) break; @@ -880,20 +904,20 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & { case GroupByKind::GROUPING_SETS: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), keys_info.grouping_set_keys)), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), keys_info.grouping_set_keys, data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), { "__grouping_set" }, column_name); break; } case GroupByKind::ROLLUP: - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number)), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number, data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), { "__grouping_set" }, column_name); break; case GroupByKind::CUBE: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number)), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number, data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), { "__grouping_set" }, column_name); break; } case GroupByKind::ORDINARY: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes))), {}, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), {}, column_name); break; } default: diff --git a/src/Interpreters/AggregationUtils.cpp b/src/Interpreters/AggregationUtils.cpp index 43062546450..ed5e1512a1f 100644 --- a/src/Interpreters/AggregationUtils.cpp +++ b/src/Interpreters/AggregationUtils.cpp @@ -50,19 +50,15 @@ OutputBlockColumns prepareOutputBlockColumns( if (aggregate_functions[i]->isState()) { - /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. - if (auto * column_aggregate_func = typeid_cast(final_aggregate_columns[i].get())) - for (auto & pool : aggregates_pools) - column_aggregate_func->addArena(pool); - - /// Aggregate state can be wrapped into array if aggregate function ends with -Resample combinator. - final_aggregate_columns[i]->forEachSubcolumn( - [&aggregates_pools](auto & subcolumn) - { - if (auto * column_aggregate_func = typeid_cast(subcolumn.get())) - for (auto & pool : aggregates_pools) - column_aggregate_func->addArena(pool); - }); + auto callback = [&](auto & subcolumn) + { + /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. + if (auto * column_aggregate_func = typeid_cast(subcolumn.get())) + for (auto & pool : aggregates_pools) + column_aggregate_func->addArena(pool); + }; + callback(final_aggregate_columns[i]); + final_aggregate_columns[i]->forEachSubcolumnRecursively(callback); } } } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index ef55f92f63a..2080f36a531 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1754,8 +1754,11 @@ inline void Aggregator::insertAggregatesIntoColumns(Mapped & mapped, MutableColu * It is also tricky, because there are aggregate functions with "-State" modifier. * When we call "insertResultInto" for them, they insert a pointer to the state to ColumnAggregateFunction * and ColumnAggregateFunction will take ownership of this state. - * So, for aggregate functions with "-State" modifier, the state must not be destroyed - * after it has been transferred to ColumnAggregateFunction. + * So, for aggregate functions with "-State" modifier, only states of all combinators that are used + * after -State will be destroyed after result has been transferred to ColumnAggregateFunction. + * For example, if we have function `uniqStateForEachMap` after aggregation we should destroy all states that + * were created by combinators `-ForEach` and `-Map`, because resulting ColumnAggregateFunction will be + * responsible only for destruction of the states created by `uniq` function. * But we should mark that the data no longer owns these states. */ @@ -1778,8 +1781,8 @@ inline void Aggregator::insertAggregatesIntoColumns(Mapped & mapped, MutableColu /** Destroy states that are no longer needed. This loop does not throw. * - * Don't destroy states for "-State" aggregate functions, - * because the ownership of this state is transferred to ColumnAggregateFunction + * For functions with -State combinator we destroy only states of all combinators that are used + * after -State, because the ownership of the rest states is transferred to ColumnAggregateFunction * and ColumnAggregateFunction will take care. * * But it's only for states that has been transferred to ColumnAggregateFunction @@ -1787,10 +1790,10 @@ inline void Aggregator::insertAggregatesIntoColumns(Mapped & mapped, MutableColu */ for (size_t destroy_i = 0; destroy_i < params.aggregates_size; ++destroy_i) { - /// If ownership was not transferred to ColumnAggregateFunction. - if (!(destroy_i < insert_i && aggregate_functions[destroy_i]->isState())) - aggregate_functions[destroy_i]->destroy( - mapped + offsets_of_aggregate_states[destroy_i]); + if (destroy_i < insert_i) + aggregate_functions[destroy_i]->destroyUpToState(mapped + offsets_of_aggregate_states[destroy_i]); + else + aggregate_functions[destroy_i]->destroy(mapped + offsets_of_aggregate_states[destroy_i]); } /// Mark the cell as destroyed so it will not be destroyed in destructor. @@ -1855,12 +1858,7 @@ Block Aggregator::insertResultsIntoColumns(PaddedPODArray & pl size_t destroy_index = aggregate_functions_destroy_index; ++aggregate_functions_destroy_index; - /// For State AggregateFunction ownership of aggregate place is passed to result column after insert - bool is_state = aggregate_functions[destroy_index]->isState(); - bool destroy_place_after_insert = !is_state; - - aggregate_functions[destroy_index]->insertResultIntoBatch( - 0, places.size(), places.data(), offset, *final_aggregate_column, arena, destroy_place_after_insert); + aggregate_functions[destroy_index]->insertResultIntoBatch(0, places.size(), places.data(), offset, *final_aggregate_column, arena); } } catch (...) diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index ae39c7035dd..cad2200c5ec 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -141,7 +141,13 @@ AsynchronousInsertQueue::~AsynchronousInsertQueue() { /// TODO: add a setting for graceful shutdown. - shutdown = true; + LOG_TRACE(log, "Shutting down the asynchronous insertion queue"); + + { + std::lock_guard lock(shutdown_mutex); + shutdown = true; + shutdown_cv.notify_all(); + } assert(dump_by_first_update_thread.joinable()); dump_by_first_update_thread.join(); @@ -162,6 +168,8 @@ AsynchronousInsertQueue::~AsynchronousInsertQueue() ErrorCodes::TIMEOUT_EXCEEDED, "Wait for async insert timeout exceeded)"))); } + + LOG_TRACE(log, "Asynchronous insertion queue finished"); } void AsynchronousInsertQueue::scheduleDataProcessingJob(const InsertQuery & key, InsertDataPtr data, ContextPtr global_context) @@ -276,10 +284,8 @@ void AsynchronousInsertQueue::busyCheck() { auto timeout = busy_timeout; - while (!shutdown) + while (!waitForShutdown(timeout)) { - std::this_thread::sleep_for(timeout); - /// TODO: use priority queue instead of raw unsorted queue. timeout = busy_timeout; std::shared_lock read_lock(rwlock); @@ -301,9 +307,8 @@ void AsynchronousInsertQueue::busyCheck() void AsynchronousInsertQueue::staleCheck() { - while (!shutdown) + while (!waitForShutdown(stale_timeout)) { - std::this_thread::sleep_for(stale_timeout); std::shared_lock read_lock(rwlock); for (auto & [key, elem] : queue) @@ -325,9 +330,8 @@ void AsynchronousInsertQueue::cleanup() /// because it holds exclusive lock. auto timeout = busy_timeout * 5; - while (!shutdown) + while (!waitForShutdown(timeout)) { - std::this_thread::sleep_for(timeout); std::vector keys_to_remove; { @@ -379,6 +383,12 @@ void AsynchronousInsertQueue::cleanup() } } +bool AsynchronousInsertQueue::waitForShutdown(const Milliseconds & timeout) +{ + std::unique_lock shutdown_lock(shutdown_mutex); + return shutdown_cv.wait_for(shutdown_lock, timeout, [this]() { return shutdown; }); +} + // static void AsynchronousInsertQueue::processData(InsertQuery key, InsertDataPtr data, ContextPtr global_context) try @@ -447,17 +457,20 @@ try format->addBuffer(std::move(last_buffer)); - auto chunk = Chunk(executor.getResultColumns(), total_rows); - size_t total_bytes = chunk.bytes(); + if (total_rows) + { + auto chunk = Chunk(executor.getResultColumns(), total_rows); + size_t total_bytes = chunk.bytes(); - auto source = std::make_shared(header, std::move(chunk)); - pipeline.complete(Pipe(std::move(source))); + auto source = std::make_shared(header, std::move(chunk)); + pipeline.complete(Pipe(std::move(source))); - CompletedPipelineExecutor completed_executor(pipeline); - completed_executor.execute(); + CompletedPipelineExecutor completed_executor(pipeline); + completed_executor.execute(); - LOG_INFO(log, "Flushed {} rows, {} bytes for query '{}'", - total_rows, total_bytes, queryToString(key.query)); + LOG_INFO(log, "Flushed {} rows, {} bytes for query '{}'", + total_rows, total_bytes, queryToString(key.query)); + } for (const auto & entry : data->entries) if (!entry->isFinished()) diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 8a4e8dad8dd..6d9aeb7f55d 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -115,7 +115,10 @@ private: const Milliseconds busy_timeout; const Milliseconds stale_timeout; - std::atomic shutdown{false}; + std::mutex shutdown_mutex; + std::condition_variable shutdown_cv; + bool shutdown{false}; + ThreadPool pool; /// dump the data only inside this pool. ThreadFromGlobalPool dump_by_first_update_thread; /// uses busy_timeout and busyCheck() ThreadFromGlobalPool dump_by_last_update_thread; /// uses stale_timeout and staleCheck() @@ -136,6 +139,10 @@ private: template static void finishWithException(const ASTPtr & query, const std::list & entries, const E & exception); + /// @param timeout - time to wait + /// @return true if shutdown requested + bool waitForShutdown(const Milliseconds & timeout); + public: auto getQueueLocked() const { diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index bccfa8f5b1e..a40b1bbcbe9 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -33,13 +33,6 @@ # include #endif - -namespace CurrentMetrics -{ - extern const Metric MemoryTracking; -} - - namespace DB { @@ -393,7 +386,7 @@ uint64_t updateJemallocEpoch() } template -static void saveJemallocMetricImpl(AsynchronousMetricValues & values, +static Value saveJemallocMetricImpl(AsynchronousMetricValues & values, const std::string & jemalloc_full_name, const std::string & clickhouse_full_name) { @@ -401,22 +394,23 @@ static void saveJemallocMetricImpl(AsynchronousMetricValues & values, size_t size = sizeof(value); mallctl(jemalloc_full_name.c_str(), &value, &size, nullptr, 0); values[clickhouse_full_name] = value; + return value; } template -static void saveJemallocMetric(AsynchronousMetricValues & values, +static Value saveJemallocMetric(AsynchronousMetricValues & values, const std::string & metric_name) { - saveJemallocMetricImpl(values, + return saveJemallocMetricImpl(values, fmt::format("stats.{}", metric_name), fmt::format("jemalloc.{}", metric_name)); } template -static void saveAllArenasMetric(AsynchronousMetricValues & values, +static Value saveAllArenasMetric(AsynchronousMetricValues & values, const std::string & metric_name) { - saveJemallocMetricImpl(values, + return saveJemallocMetricImpl(values, fmt::format("stats.arenas.{}.{}", MALLCTL_ARENAS_ALL, metric_name), fmt::format("jemalloc.arenas.all.{}", metric_name)); } @@ -657,10 +651,39 @@ void AsynchronousMetrics::update(TimePoint update_time) } } +#if defined(OS_LINUX) || defined(OS_FREEBSD) + MemoryStatisticsOS::Data memory_statistics_data = memory_stat.get(); +#endif + +#if USE_JEMALLOC + // 'epoch' is a special mallctl -- it updates the statistics. Without it, all + // the following calls will return stale values. It increments and returns + // the current epoch number, which might be useful to log as a sanity check. + auto epoch = updateJemallocEpoch(); + new_values["jemalloc.epoch"] = epoch; + + // Collect the statistics themselves. + saveJemallocMetric(new_values, "allocated"); + saveJemallocMetric(new_values, "active"); + saveJemallocMetric(new_values, "metadata"); + saveJemallocMetric(new_values, "metadata_thp"); + saveJemallocMetric(new_values, "resident"); + saveJemallocMetric(new_values, "mapped"); + saveJemallocMetric(new_values, "retained"); + saveJemallocMetric(new_values, "background_thread.num_threads"); + saveJemallocMetric(new_values, "background_thread.num_runs"); + saveJemallocMetric(new_values, "background_thread.run_intervals"); + saveAllArenasMetric(new_values, "pactive"); + [[maybe_unused]] size_t je_malloc_pdirty = saveAllArenasMetric(new_values, "pdirty"); + [[maybe_unused]] size_t je_malloc_pmuzzy = saveAllArenasMetric(new_values, "pmuzzy"); + saveAllArenasMetric(new_values, "dirty_purged"); + saveAllArenasMetric(new_values, "muzzy_purged"); +#endif + /// Process process memory usage according to OS #if defined(OS_LINUX) || defined(OS_FREEBSD) { - MemoryStatisticsOS::Data data = memory_stat.get(); + MemoryStatisticsOS::Data & data = memory_statistics_data; new_values["MemoryVirtual"] = data.virt; new_values["MemoryResident"] = data.resident; @@ -676,9 +699,16 @@ void AsynchronousMetrics::update(TimePoint update_time) { Int64 amount = total_memory_tracker.get(); Int64 peak = total_memory_tracker.getPeak(); - Int64 new_amount = data.resident; + Int64 rss = data.resident; + Int64 free_memory_in_allocator_arenas = 0; - Int64 difference = new_amount - amount; +#if USE_JEMALLOC + /// This is a memory which is kept by allocator. + /// Will subsract it from RSS to decrease memory drift. + free_memory_in_allocator_arenas = je_malloc_pdirty * getPageSize(); +#endif + + Int64 difference = rss - free_memory_in_allocator_arenas - amount; /// Log only if difference is high. This is for convenience. The threshold is arbitrary. if (difference >= 1048576 || difference <= -1048576) @@ -686,11 +716,10 @@ void AsynchronousMetrics::update(TimePoint update_time) "MemoryTracking: was {}, peak {}, will set to {} (RSS), difference: {}", ReadableSize(amount), ReadableSize(peak), - ReadableSize(new_amount), + ReadableSize(rss), ReadableSize(difference)); - total_memory_tracker.set(new_amount); - CurrentMetrics::set(CurrentMetrics::MemoryTracking, new_amount); + total_memory_tracker.setRSS(rss, free_memory_in_allocator_arenas); } } #endif @@ -1561,31 +1590,6 @@ void AsynchronousMetrics::update(TimePoint update_time) } #endif -#if USE_JEMALLOC - // 'epoch' is a special mallctl -- it updates the statistics. Without it, all - // the following calls will return stale values. It increments and returns - // the current epoch number, which might be useful to log as a sanity check. - auto epoch = updateJemallocEpoch(); - new_values["jemalloc.epoch"] = epoch; - - // Collect the statistics themselves. - saveJemallocMetric(new_values, "allocated"); - saveJemallocMetric(new_values, "active"); - saveJemallocMetric(new_values, "metadata"); - saveJemallocMetric(new_values, "metadata_thp"); - saveJemallocMetric(new_values, "resident"); - saveJemallocMetric(new_values, "mapped"); - saveJemallocMetric(new_values, "retained"); - saveJemallocMetric(new_values, "background_thread.num_threads"); - saveJemallocMetric(new_values, "background_thread.num_runs"); - saveJemallocMetric(new_values, "background_thread.run_intervals"); - saveAllArenasMetric(new_values, "pactive"); - saveAllArenasMetric(new_values, "pdirty"); - saveAllArenasMetric(new_values, "pmuzzy"); - saveAllArenasMetric(new_values, "dirty_purged"); - saveAllArenasMetric(new_values, "muzzy_purged"); -#endif - updateHeavyMetricsIfNeeded(current_time, update_time, new_values); /// Add more metrics as you wish. @@ -1665,7 +1669,7 @@ void AsynchronousMetrics::updateHeavyMetricsIfNeeded(TimePoint current_time, Tim LOG_IMPL(log, log_level.first, log_level.second, "Update heavy metrics. " "Update period {} sec. " - "Update heavy metrics period {} sec. " + "Update heavy metrics period {} sec. " "Heavy metrics calculation elapsed: {} sec.", update_period.count(), heavy_metric_update_period.count(), diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index f51df9ae737..20a9f6cce1d 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -122,7 +122,6 @@ void FileCache::initialize() fs::create_directories(cache_base_path); } - status_file = make_unique(fs::path(cache_base_path) / "status", StatusFile::write_full_info); is_initialized = true; } } @@ -258,7 +257,7 @@ FileSegments FileCache::splitRangeIntoCells( size_t offset, size_t size, FileSegment::State state, - bool is_persistent, + const CreateFileSegmentSettings & settings, std::lock_guard & cache_lock) { assert(size > 0); @@ -275,7 +274,7 @@ FileSegments FileCache::splitRangeIntoCells( current_cell_size = std::min(remaining_size, max_file_segment_size); remaining_size -= current_cell_size; - auto * cell = addCell(key, current_pos, current_cell_size, state, is_persistent, cache_lock); + auto * cell = addCell(key, current_pos, current_cell_size, state, settings, cache_lock); if (cell) file_segments.push_back(cell->file_segment); assert(cell); @@ -292,7 +291,7 @@ void FileCache::fillHolesWithEmptyFileSegments( const Key & key, const FileSegment::Range & range, bool fill_with_detached_file_segments, - bool is_persistent, + const CreateFileSegmentSettings & settings, std::lock_guard & cache_lock) { /// There are segments [segment1, ..., segmentN] @@ -339,16 +338,16 @@ void FileCache::fillHolesWithEmptyFileSegments( if (fill_with_detached_file_segments) { - auto file_segment = std::make_shared(current_pos, hole_size, key, this, FileSegment::State::EMPTY); + auto file_segment = std::make_shared(current_pos, hole_size, key, this, FileSegment::State::EMPTY, settings); { - std::lock_guard segment_lock(file_segment->mutex); - file_segment->markAsDetached(segment_lock); + std::unique_lock segment_lock(file_segment->mutex); + file_segment->detachAssumeStateFinalized(segment_lock); } file_segments.insert(it, file_segment); } else { - file_segments.splice(it, splitRangeIntoCells(key, current_pos, hole_size, FileSegment::State::EMPTY, is_persistent, cache_lock)); + file_segments.splice(it, splitRangeIntoCells(key, current_pos, hole_size, FileSegment::State::EMPTY, settings, cache_lock)); } current_pos = segment_range.right + 1; @@ -366,22 +365,23 @@ void FileCache::fillHolesWithEmptyFileSegments( if (fill_with_detached_file_segments) { - auto file_segment = std::make_shared(current_pos, hole_size, key, this, FileSegment::State::EMPTY); + auto file_segment = std::make_shared(current_pos, hole_size, key, this, FileSegment::State::EMPTY, settings); { - std::lock_guard segment_lock(file_segment->mutex); - file_segment->markAsDetached(segment_lock); + std::unique_lock segment_lock(file_segment->mutex); + file_segment->detachAssumeStateFinalized(segment_lock); } file_segments.insert(file_segments.end(), file_segment); } else { file_segments.splice( - file_segments.end(), splitRangeIntoCells(key, current_pos, hole_size, FileSegment::State::EMPTY, is_persistent, cache_lock)); + file_segments.end(), + splitRangeIntoCells(key, current_pos, hole_size, FileSegment::State::EMPTY, settings, cache_lock)); } } } -FileSegmentsHolder FileCache::getOrSet(const Key & key, size_t offset, size_t size, bool is_persistent) +FileSegmentsHolder FileCache::getOrSet(const Key & key, size_t offset, size_t size, const CreateFileSegmentSettings & settings) { std::lock_guard cache_lock(mutex); @@ -398,11 +398,11 @@ FileSegmentsHolder FileCache::getOrSet(const Key & key, size_t offset, size_t si if (file_segments.empty()) { - file_segments = splitRangeIntoCells(key, offset, size, FileSegment::State::EMPTY, is_persistent, cache_lock); + file_segments = splitRangeIntoCells(key, offset, size, FileSegment::State::EMPTY, settings, cache_lock); } else { - fillHolesWithEmptyFileSegments(file_segments, key, range, /* fill_with_detached */false, is_persistent, cache_lock); + fillHolesWithEmptyFileSegments(file_segments, key, range, /* fill_with_detached */false, settings, cache_lock); } assert(!file_segments.empty()); @@ -426,16 +426,17 @@ FileSegmentsHolder FileCache::get(const Key & key, size_t offset, size_t size) if (file_segments.empty()) { - auto file_segment = std::make_shared(offset, size, key, this, FileSegment::State::EMPTY); + auto file_segment = std::make_shared( + offset, size, key, this, FileSegment::State::EMPTY, CreateFileSegmentSettings{}); { - std::lock_guard segment_lock(file_segment->mutex); - file_segment->markAsDetached(segment_lock); + std::unique_lock segment_lock(file_segment->mutex); + file_segment->detachAssumeStateFinalized(segment_lock); } file_segments = { file_segment }; } else { - fillHolesWithEmptyFileSegments(file_segments, key, range, /* fill_with_detached */true, /* is_persistent */false, cache_lock); + fillHolesWithEmptyFileSegments(file_segments, key, range, /* fill_with_detached */true, {}, cache_lock); } return FileSegmentsHolder(std::move(file_segments)); @@ -443,7 +444,7 @@ FileSegmentsHolder FileCache::get(const Key & key, size_t offset, size_t size) FileCache::FileSegmentCell * FileCache::addCell( const Key & key, size_t offset, size_t size, - FileSegment::State state, bool is_persistent, + FileSegment::State state, const CreateFileSegmentSettings & settings, std::lock_guard & cache_lock) { /// Create a file segment cell and put it in `files` map by [key][offset]. @@ -475,18 +476,23 @@ FileCache::FileSegmentCell * FileCache::addCell( stash_records.erase({remove_priority_iter->key(), remove_priority_iter->offset()}); remove_priority_iter->removeAndGetNext(cache_lock); } - /// For segments that do not reach the download threshold, we do not download them, but directly read them + + /// For segments that do not reach the download threshold, + /// we do not download them, but directly read them result_state = FileSegment::State::SKIP_CACHE; } else { auto priority_iter = record->second; priority_iter->use(cache_lock); - result_state = priority_iter->hits() >= enable_cache_hits_threshold ? FileSegment::State::EMPTY : FileSegment::State::SKIP_CACHE; + + result_state = priority_iter->hits() >= enable_cache_hits_threshold + ? FileSegment::State::EMPTY + : FileSegment::State::SKIP_CACHE; } } - return std::make_shared(offset, size, key, this, result_state, is_persistent); + return std::make_shared(offset, size, key, this, result_state, settings); }; FileSegmentCell cell(skip_or_download(), this, cache_lock); @@ -495,6 +501,7 @@ FileCache::FileSegmentCell * FileCache::addCell( if (offsets.empty()) { auto key_path = getPathInLocalCache(key); + if (!fs::exists(key_path)) fs::create_directories(key_path); } @@ -513,7 +520,7 @@ FileSegmentPtr FileCache::createFileSegmentForDownload( const Key & key, size_t offset, size_t size, - bool is_persistent, + const CreateFileSegmentSettings & settings, std::lock_guard & cache_lock) { #ifndef NDEBUG @@ -530,7 +537,7 @@ FileSegmentPtr FileCache::createFileSegmentForDownload( "Cache cell already exists for key `{}` and offset {}", key.toString(), offset); - cell = addCell(key, offset, size, FileSegment::State::EMPTY, is_persistent, cache_lock); + cell = addCell(key, offset, size, FileSegment::State::EMPTY, settings, cache_lock); if (!cell) throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to add a new cell for download"); @@ -542,18 +549,21 @@ bool FileCache::tryReserve(const Key & key, size_t offset, size_t size, std::loc { auto query_context = enable_filesystem_query_cache_limit ? getCurrentQueryContext(cache_lock) : nullptr; if (!query_context) + { return tryReserveForMainList(key, offset, size, nullptr, cache_lock); - + } /// The maximum cache capacity of the request is not reached, thus the //// cache block is evicted from the main LRU queue by tryReserveForMainList(). else if (query_context->getCacheSize() + size <= query_context->getMaxCacheSize()) + { return tryReserveForMainList(key, offset, size, query_context, cache_lock); - + } /// When skip_download_if_exceeds_query_cache is true, there is no need /// to evict old data, skip the cache and read directly from remote fs. else if (query_context->isSkipDownloadIfExceed()) + { return false; - + } /// The maximum cache size of the query is reached, the cache will be /// evicted from the history cache accessed by the current query. else @@ -833,7 +843,7 @@ void FileCache::removeIfExists(const Key & key) auto file_segment = cell->file_segment; if (file_segment) { - std::lock_guard segment_lock(file_segment->mutex); + std::unique_lock segment_lock(file_segment->mutex); file_segment->detach(cache_lock, segment_lock); remove(file_segment->key(), file_segment->offset(), cache_lock, segment_lock); } @@ -863,9 +873,11 @@ void FileCache::removeIfReleasable() auto * cell = getCell(key, offset, cache_lock); if (!cell) + { throw Exception( ErrorCodes::LOGICAL_ERROR, "Cache is in inconsistent state: LRU queue contains entries with no cache cell"); + } if (cell->releasable()) { @@ -880,7 +892,7 @@ void FileCache::removeIfReleasable() for (auto & file_segment : to_remove) { - std::lock_guard segment_lock(file_segment->mutex); + std::unique_lock segment_lock(file_segment->mutex); file_segment->detach(cache_lock, segment_lock); remove(file_segment->key(), file_segment->offset(), cache_lock, segment_lock); } @@ -896,13 +908,13 @@ void FileCache::removeIfReleasable() void FileCache::remove(FileSegmentPtr file_segment, std::lock_guard & cache_lock) { - std::lock_guard segment_lock(file_segment->mutex); + std::unique_lock segment_lock(file_segment->mutex); remove(file_segment->key(), file_segment->offset(), cache_lock, segment_lock); } void FileCache::remove( Key key, size_t offset, - std::lock_guard & cache_lock, std::lock_guard & /* segment_lock */) + std::lock_guard & cache_lock, std::unique_lock & /* segment_lock */) { LOG_DEBUG(log, "Remove from cache. Key: {}, offset: {}", key.toString(), offset); @@ -976,7 +988,7 @@ void FileCache::loadCacheInfoIntoMemory(std::lock_guard & cache_lock { if (!key_it->is_directory()) { - LOG_DEBUG(log, "Unexpected file {} (not a directory), will skip it", key_it->path().string()); + LOG_DEBUG(log, "Unexpected file: {}. Expected a directory", key_it->path().string()); continue; } @@ -1012,7 +1024,10 @@ void FileCache::loadCacheInfoIntoMemory(std::lock_guard & cache_lock if (tryReserve(key, offset, size, cache_lock)) { - auto * cell = addCell(key, offset, size, FileSegment::State::DOWNLOADED, is_persistent, cache_lock); + auto * cell = addCell( + key, offset, size, FileSegment::State::DOWNLOADED, + CreateFileSegmentSettings{ .is_persistent = is_persistent }, cache_lock); + if (cell) queue_entries.emplace_back(cell->queue_iterator, cell->file_segment); } @@ -1049,7 +1064,7 @@ void FileCache::loadCacheInfoIntoMemory(std::lock_guard & cache_lock void FileCache::reduceSizeToDownloaded( const Key & key, size_t offset, - std::lock_guard & cache_lock, std::lock_guard & /* segment_lock */) + std::lock_guard & cache_lock, std::unique_lock & segment_lock) { /** * In case file was partially downloaded and it's download cannot be continued @@ -1069,20 +1084,25 @@ void FileCache::reduceSizeToDownloaded( const auto & file_segment = cell->file_segment; size_t downloaded_size = file_segment->downloaded_size; - if (downloaded_size == file_segment->range().size()) + size_t full_size = file_segment->range().size(); + + if (downloaded_size == full_size) { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Nothing to reduce, file segment fully downloaded, key: {}, offset: {}", - key.toString(), offset); + "Nothing to reduce, file segment fully downloaded: {}", + file_segment->getInfoForLogUnlocked(segment_lock)); } - cell->file_segment = std::make_shared(offset, downloaded_size, key, this, FileSegment::State::DOWNLOADED); + cell->file_segment = std::make_shared( + offset, downloaded_size, key, this, FileSegment::State::DOWNLOADED, CreateFileSegmentSettings{}); + + assert(file_segment->reserved_size == downloaded_size); } bool FileCache::isLastFileSegmentHolder( const Key & key, size_t offset, - std::lock_guard & cache_lock, std::lock_guard & /* segment_lock */) + std::lock_guard & cache_lock, std::unique_lock & /* segment_lock */) { auto * cell = getCell(key, offset, cache_lock); @@ -1167,7 +1187,8 @@ FileCache::FileSegmentCell::FileSegmentCell( { case FileSegment::State::DOWNLOADED: { - queue_iterator = cache->main_priority->add(file_segment->key(), file_segment->offset(), file_segment->range().size(), cache_lock); + queue_iterator = cache->main_priority->add( + file_segment->key(), file_segment->offset(), file_segment->range().size(), cache_lock); break; } case FileSegment::State::SKIP_CACHE: @@ -1246,14 +1267,41 @@ void FileCache::assertPriorityCorrectness(std::lock_guard & cache_lo ErrorCodes::LOGICAL_ERROR, "Cache is in inconsistent state: LRU queue contains entries with no cache cell (assertCorrectness())"); } - assert(cell->size() == size); + + if (cell->size() != size) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected {} == {} size ({})", + cell->size(), size, cell->file_segment->getInfoForLog()); + } + total_size += size; } + assert(total_size == main_priority->getCacheSize(cache_lock)); assert(main_priority->getCacheSize(cache_lock) <= max_size); assert(main_priority->getElementsNum(cache_lock) <= max_element_size); } +FileCache::QueryContextHolder::QueryContextHolder( + const String & query_id_, + FileCache * cache_, + FileCache::QueryContextPtr context_) + : query_id(query_id_) + , cache(cache_) + , context(context_) +{ +} + +FileCache::QueryContextHolder::~QueryContextHolder() +{ + /// If only the query_map and the current holder hold the context_query, + /// the query has been completed and the query_context is released. + if (context && context.use_count() == 2) + cache->removeQueryContext(query_id); +} + FileCache::QueryContextPtr FileCache::getCurrentQueryContext(std::lock_guard & cache_lock) { if (!isQueryInitialized()) @@ -1362,22 +1410,4 @@ void FileCache::QueryContext::use(const Key & key, size_t offset, std::lock_guar record->second->use(cache_lock); } -FileCache::QueryContextHolder::QueryContextHolder( - const String & query_id_, - FileCache * cache_, - FileCache::QueryContextPtr context_) - : query_id(query_id_) - , cache(cache_) - , context(context_) -{ -} - -FileCache::QueryContextHolder::~QueryContextHolder() -{ - /// If only the query_map and the current holder hold the context_query, - /// the query has been completed and the query_context is released. - if (context && context.use_count() == 2) - cache->removeQueryContext(query_id); -} - } diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 3f5a5c9e1c5..07aea230803 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -12,13 +12,14 @@ #include #include +#include +#include #include +#include +#include #include #include -#include -#include -#include -#include + namespace DB { @@ -43,7 +44,6 @@ public: ~FileCache() = default; - /// Restore cache from local filesystem. void initialize(); const String & getBasePath() const { return cache_base_path; } @@ -59,7 +59,7 @@ public: * As long as pointers to returned file segments are hold * it is guaranteed that these file segments are not removed from cache. */ - FileSegmentsHolder getOrSet(const Key & key, size_t offset, size_t size, bool is_persistent); + FileSegmentsHolder getOrSet(const Key & key, size_t offset, size_t size, const CreateFileSegmentSettings & settings); /** * Segments in returned list are ordered in ascending order and represent a full contiguous @@ -104,7 +104,7 @@ public: const Key & key, size_t offset, size_t size, - bool is_persistent, + const CreateFileSegmentSettings & create_settings, std::lock_guard & cache_lock); FileSegments getSnapshot() const; @@ -132,21 +132,21 @@ public: private: String cache_base_path; - size_t max_size; - size_t max_element_size; - size_t max_file_segment_size; + const size_t max_size; + const size_t max_element_size; + const size_t max_file_segment_size; - bool allow_persistent_files; - size_t enable_cache_hits_threshold; - bool enable_filesystem_query_cache_limit; + const bool allow_persistent_files; + const size_t enable_cache_hits_threshold; + const bool enable_filesystem_query_cache_limit; + mutable std::mutex mutex; Poco::Logger * log; bool is_initialized = false; std::exception_ptr initialization_exception; - std::unique_ptr status_file; - mutable std::mutex mutex; + void assertInitialized(std::lock_guard & cache_lock) const; bool tryReserve(const Key & key, size_t offset, size_t size, std::lock_guard & cache_lock); @@ -154,7 +154,7 @@ private: Key key, size_t offset, std::lock_guard & cache_lock, - std::lock_guard & segment_lock); + std::unique_lock & segment_lock); void remove( FileSegmentPtr file_segment, @@ -164,15 +164,13 @@ private: const Key & key, size_t offset, std::lock_guard & cache_lock, - std::lock_guard & segment_lock); + std::unique_lock & segment_lock); void reduceSizeToDownloaded( const Key & key, size_t offset, std::lock_guard & cache_lock, - std::lock_guard & segment_lock); - - void assertInitialized(std::lock_guard & cache_lock) const; + std::unique_lock & segment_lock); struct FileSegmentCell : private boost::noncopyable { @@ -225,7 +223,7 @@ private: size_t offset, size_t size, FileSegment::State state, - bool is_persistent, + const CreateFileSegmentSettings & create_settings, std::lock_guard & cache_lock); static void useCell(const FileSegmentCell & cell, FileSegments & result, std::lock_guard & cache_lock); @@ -242,7 +240,7 @@ private: size_t offset, size_t size, FileSegment::State state, - bool is_persistent, + const CreateFileSegmentSettings & create_settings, std::lock_guard & cache_lock); String dumpStructureUnlocked(const Key & key_, std::lock_guard & cache_lock); @@ -252,7 +250,7 @@ private: const Key & key, const FileSegment::Range & range, bool fill_with_detached_file_segments, - bool is_persistent, + const CreateFileSegmentSettings & settings, std::lock_guard & cache_lock); size_t getUsedCacheSizeUnlocked(std::lock_guard & cache_lock) const; diff --git a/src/Interpreters/Cache/FileCache_fwd.h b/src/Interpreters/Cache/FileCache_fwd.h index 13f037783b0..25c16b4e840 100644 --- a/src/Interpreters/Cache/FileCache_fwd.h +++ b/src/Interpreters/Cache/FileCache_fwd.h @@ -12,5 +12,6 @@ class FileCache; using FileCachePtr = std::shared_ptr; struct FileCacheSettings; +struct CreateFileSegmentSettings; } diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 547e6849dd6..708d60f56dc 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -1,10 +1,10 @@ #include "FileSegment.h" #include -#include +#include +#include #include #include -#include #include #include #include @@ -29,7 +29,7 @@ FileSegment::FileSegment( const Key & key_, FileCache * cache_, State download_state_, - bool is_persistent_) + const CreateFileSegmentSettings & settings) : segment_range(offset_, offset_ + size_ - 1) , download_state(download_state_) , file_key(key_) @@ -39,7 +39,7 @@ FileSegment::FileSegment( #else , log(&Poco::Logger::get("FileSegment")) #endif - , is_persistent(is_persistent_) + , is_persistent(settings.is_persistent) { /// On creation, file segment state can be EMPTY, DOWNLOADED, DOWNLOADING. switch (download_state) @@ -64,50 +64,73 @@ FileSegment::FileSegment( } default: { - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Can create cell with either EMPTY, DOWNLOADED, DOWNLOADING state"); + throw Exception( + ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + "Can create cell with either EMPTY, DOWNLOADED, DOWNLOADING state"); } } } +String FileSegment::getPathInLocalCache() const +{ + return cache->getPathInLocalCache(key(), offset(), isPersistent()); +} + FileSegment::State FileSegment::state() const { - std::lock_guard segment_lock(mutex); + std::unique_lock segment_lock(mutex); return download_state; } -size_t FileSegment::getDownloadOffset() const +void FileSegment::setDownloadState(State state) +{ + LOG_TEST(log, "Updated state from {} to {}", stateToString(download_state), stateToString(state)); + download_state = state; +} + +size_t FileSegment::getFirstNonDownloadedOffset() const +{ + std::unique_lock segment_lock(mutex); + return getFirstNonDownloadedOffsetUnlocked(segment_lock); +} + +size_t FileSegment::getFirstNonDownloadedOffsetUnlocked(std::unique_lock & segment_lock) const { - std::lock_guard segment_lock(mutex); return range().left + getDownloadedSizeUnlocked(segment_lock); } +size_t FileSegment::getCurrentWriteOffset() const +{ + std::unique_lock segment_lock(mutex); + return getCurrentWriteOffsetUnlocked(segment_lock); +} + +size_t FileSegment::getCurrentWriteOffsetUnlocked(std::unique_lock & segment_lock) const +{ + return getFirstNonDownloadedOffsetUnlocked(segment_lock); +} + size_t FileSegment::getDownloadedSize() const { - std::lock_guard segment_lock(mutex); + std::unique_lock segment_lock(mutex); return getDownloadedSizeUnlocked(segment_lock); } -size_t FileSegment::getRemainingSizeToDownload() const -{ - std::lock_guard segment_lock(mutex); - return range().size() - getDownloadedSizeUnlocked(segment_lock); -} - -bool FileSegment::isDetached() const -{ - std::lock_guard segment_lock(mutex); - return is_detached; -} - -size_t FileSegment::getDownloadedSizeUnlocked(std::lock_guard & /* segment_lock */) const +size_t FileSegment::getDownloadedSizeUnlocked(std::unique_lock & /* segment_lock */) const { if (download_state == State::DOWNLOADED) return downloaded_size; - std::lock_guard download_lock(download_mutex); + std::unique_lock download_lock(download_mutex); return downloaded_size; } +bool FileSegment::isDownloaded() const +{ + std::lock_guard segment_lock(mutex); + return is_downloaded; +} + String FileSegment::getCallerId() { if (!CurrentThread::isInitialized() @@ -118,84 +141,106 @@ String FileSegment::getCallerId() return std::string(CurrentThread::getQueryId()) + ":" + toString(getThreadId()); } +String FileSegment::getDownloader() const +{ + std::unique_lock segment_lock(mutex); + return getDownloaderUnlocked(segment_lock); +} + +String FileSegment::getDownloaderUnlocked(std::unique_lock & /* segment_lock */) const +{ + return downloader_id; +} + String FileSegment::getOrSetDownloader() { - std::lock_guard segment_lock(mutex); + std::unique_lock segment_lock(mutex); - assertNotDetached(segment_lock); + assertNotDetachedUnlocked(segment_lock); - if (downloader_id.empty()) + auto current_downloader = getDownloaderUnlocked(segment_lock); + + if (current_downloader.empty()) { - assert(download_state != State::DOWNLOADING); + bool allow_new_downloader = download_state == State::EMPTY || download_state == State::PARTIALLY_DOWNLOADED; + if (!allow_new_downloader) + return "notAllowed:" + stateToString(download_state); - if (download_state != State::EMPTY - && download_state != State::PARTIALLY_DOWNLOADED) - return "None"; - - downloader_id = getCallerId(); - download_state = State::DOWNLOADING; + current_downloader = downloader_id = getCallerId(); + setDownloadState(State::DOWNLOADING); } - else if (downloader_id == getCallerId()) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Attempt to set the same downloader for segment {} for the second time", range().toString()); - return downloader_id; + return current_downloader; +} + +void FileSegment::resetDownloadingStateUnlocked([[maybe_unused]] std::unique_lock & segment_lock) +{ + assert(isDownloaderUnlocked(segment_lock)); + assert(download_state == State::DOWNLOADING); + + size_t current_downloaded_size = getDownloadedSizeUnlocked(segment_lock); + /// range().size() can equal 0 in case of write-though cache. + if (current_downloaded_size != 0 && current_downloaded_size == range().size()) + setDownloadedUnlocked(segment_lock); + else + setDownloadState(State::PARTIALLY_DOWNLOADED); } void FileSegment::resetDownloader() { - std::lock_guard segment_lock(mutex); + std::unique_lock segment_lock(mutex); - assertNotDetached(segment_lock); + assertNotDetachedUnlocked(segment_lock); + assertIsDownloaderUnlocked("resetDownloader", segment_lock); - if (downloader_id.empty()) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "There is no downloader"); - - if (getCallerId() != downloader_id) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Downloader can be reset only by downloader"); - - resetDownloaderImpl(segment_lock); + resetDownloadingStateUnlocked(segment_lock); + resetDownloaderUnlocked(segment_lock); } -void FileSegment::resetDownloaderImpl(std::lock_guard & segment_lock) +void FileSegment::resetDownloaderUnlocked(std::unique_lock & /* segment_lock */) { - if (getDownloadedSizeUnlocked(segment_lock) == range().size()) - setDownloaded(segment_lock); - else - download_state = State::PARTIALLY_DOWNLOADED; - + LOG_TEST(log, "Resetting downloader from {}", downloader_id); downloader_id.clear(); } -String FileSegment::getDownloader() const +void FileSegment::assertIsDownloaderUnlocked(const std::string & operation, std::unique_lock & segment_lock) const { - std::lock_guard segment_lock(mutex); - return downloader_id; + auto caller = getCallerId(); + auto current_downloader = getDownloaderUnlocked(segment_lock); + LOG_TEST(log, "Downloader id: {}, caller id: {}", current_downloader, caller); + + if (caller != current_downloader) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Operation `{}` can be done only by downloader. " + "(CallerId: {}, downloader id: {})", + operation, caller, downloader_id); + } } bool FileSegment::isDownloader() const { - std::lock_guard segment_lock(mutex); - return getCallerId() == downloader_id; + std::unique_lock segment_lock(mutex); + return isDownloaderUnlocked(segment_lock); } -bool FileSegment::isDownloaderImpl(std::lock_guard & /* segment_lock */) const +bool FileSegment::isDownloaderUnlocked(std::unique_lock & segment_lock) const { - return getCallerId() == downloader_id; + return getCallerId() == getDownloaderUnlocked(segment_lock); } FileSegment::RemoteFileReaderPtr FileSegment::getRemoteFileReader() { - if (!isDownloader()) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Only downloader can use remote filesystem file reader"); - + std::unique_lock segment_lock(mutex); + assertIsDownloaderUnlocked("getRemoteFileReader", segment_lock); return remote_file_reader; } FileSegment::RemoteFileReaderPtr FileSegment::extractRemoteFileReader() { std::lock_guard cache_lock(cache->mutex); - std::lock_guard segment_lock(mutex); + std::unique_lock segment_lock(mutex); if (!is_detached) { @@ -210,8 +255,8 @@ FileSegment::RemoteFileReaderPtr FileSegment::extractRemoteFileReader() void FileSegment::setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_) { - if (!isDownloader()) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Only downloader can use remote filesystem file reader"); + std::unique_lock segment_lock(mutex); + assertIsDownloaderUnlocked("setRemoteFileReader", segment_lock); if (remote_file_reader) throw Exception(ErrorCodes::LOGICAL_ERROR, "Remote file reader already exists"); @@ -221,8 +266,8 @@ void FileSegment::setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_) void FileSegment::resetRemoteFileReader() { - if (!isDownloader()) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Only downloader can use remote filesystem file reader"); + std::unique_lock segment_lock(mutex); + assertIsDownloaderUnlocked("resetRemoteFileReader", segment_lock); if (!remote_file_reader) throw Exception(ErrorCodes::LOGICAL_ERROR, "Remote file reader does not exist"); @@ -230,55 +275,59 @@ void FileSegment::resetRemoteFileReader() remote_file_reader.reset(); } -void FileSegment::write(const char * from, size_t size, size_t offset_) +void FileSegment::write(const char * from, size_t size, size_t offset) { if (!size) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Writing zero size is not allowed"); - if (availableSize() < size) - throw Exception( - ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Not enough space is reserved. Available: {}, expected: {}", availableSize(), size); - - if (!isDownloader()) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Only downloader can do the downloading. (CallerId: {}, DownloaderId: {})", - getCallerId(), downloader_id); - - if (getDownloadedSize() == range().size()) - throw Exception( - ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Attempt to write {} bytes to offset: {}, but current file segment is already fully downloaded", - size, offset_); - - auto download_offset = range().left + downloaded_size; - if (offset_ != download_offset) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Attempt to write {} bytes to offset: {}, but current download offset is {}", - size, offset_, download_offset); - { - std::lock_guard segment_lock(mutex); - assertNotDetached(segment_lock); - } + std::unique_lock segment_lock(mutex); - if (!cache_writer) - { - if (downloaded_size > 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Cache writer was finalized (downloaded size: {}, state: {})", - downloaded_size, stateToString(download_state)); + assertIsDownloaderUnlocked("write", segment_lock); + assertNotDetachedUnlocked(segment_lock); - auto download_path = getPathInLocalCache(); - cache_writer = std::make_unique(download_path); + if (download_state != State::DOWNLOADING) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected DOWNLOADING state, got {}", stateToString(download_state)); + + size_t first_non_downloaded_offset = getFirstNonDownloadedOffsetUnlocked(segment_lock); + if (offset != first_non_downloaded_offset) + throw Exception( + ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + "Attempt to write {} bytes to offset: {}, but current write offset is {}", + size, offset, first_non_downloaded_offset); + + size_t current_downloaded_size = getDownloadedSizeUnlocked(segment_lock); + chassert(reserved_size >= current_downloaded_size); + size_t free_reserved_size = reserved_size - current_downloaded_size; + + if (free_reserved_size < size) + throw Exception( + ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + "Not enough space is reserved. Available: {}, expected: {}", free_reserved_size, size); + + if (current_downloaded_size == range().size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "File segment is already fully downloaded"); + + if (!cache_writer) + { + if (current_downloaded_size > 0) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cache writer was finalized (downloaded size: {}, state: {})", + current_downloaded_size, stateToString(download_state)); + + auto download_path = getPathInLocalCache(); + cache_writer = std::make_unique(download_path); + } } try { cache_writer->write(from, size); - std::lock_guard download_lock(download_mutex); + std::unique_lock download_lock(download_mutex); cache_writer->next(); @@ -286,23 +335,20 @@ void FileSegment::write(const char * from, size_t size, size_t offset_) } catch (Exception & e) { - std::lock_guard segment_lock(mutex); + std::unique_lock segment_lock(mutex); wrapWithCacheInfo(e, "while writing into cache", segment_lock); - setDownloadFailed(segment_lock); + setDownloadFailedUnlocked(segment_lock); cv.notify_all(); throw; } - assert(getDownloadOffset() == offset_ + size); -} - -String FileSegment::getPathInLocalCache() const -{ - return cache->getPathInLocalCache(key(), offset(), isPersistent()); +#ifndef NDEBUG + chassert(getFirstNonDownloadedOffset() == offset + size); +#endif } FileSegment::State FileSegment::wait() @@ -324,8 +370,8 @@ FileSegment::State FileSegment::wait() { LOG_TEST(log, "{} waiting on: {}, current downloader: {}", getCallerId(), range().toString(), downloader_id); - assert(!downloader_id.empty()); - assert(downloader_id != getCallerId()); + chassert(!getDownloaderUnlocked(segment_lock).empty()); + chassert(!isDownloaderUnlocked(segment_lock)); cv.wait_for(segment_lock, std::chrono::seconds(60)); } @@ -338,30 +384,23 @@ bool FileSegment::reserve(size_t size_to_reserve) if (!size_to_reserve) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Zero space reservation is not allowed"); + size_t expected_downloaded_size; + { - std::lock_guard segment_lock(mutex); - assertNotDetached(segment_lock); + std::unique_lock segment_lock(mutex); - auto caller_id = getCallerId(); - bool is_downloader = caller_id == downloader_id; - if (!is_downloader) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Space can be reserved only by downloader (current: {}, expected: {})", - caller_id, downloader_id); - } + assertNotDetachedUnlocked(segment_lock); + assertIsDownloaderUnlocked("reserve", segment_lock); - size_t current_downloaded_size = getDownloadedSizeUnlocked(segment_lock); - if (current_downloaded_size + size_to_reserve > range().size()) - { + expected_downloaded_size = getDownloadedSizeUnlocked(segment_lock); + + if (expected_downloaded_size + size_to_reserve > range().size()) throw Exception( ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Attempt to reserve space too much space: {} ({})", - size_to_reserve, getInfoForLogImpl(segment_lock)); - } + "Attempt to reserve space too much space ({}) for file segment with range: {} (downloaded size: {})", + size_to_reserve, range().toString(), downloaded_size); - assert(reserved_size >= current_downloaded_size); + chassert(reserved_size >= expected_downloaded_size); } /** @@ -370,9 +409,7 @@ bool FileSegment::reserve(size_t size_to_reserve) * and the caller is going to continue; */ - size_t current_downloaded_size = getDownloadedSize(); - assert(reserved_size >= current_downloaded_size); - size_t already_reserved_size = reserved_size - current_downloaded_size; + size_t already_reserved_size = reserved_size - expected_downloaded_size; bool reserved = already_reserved_size >= size_to_reserve; if (!reserved) @@ -392,23 +429,13 @@ bool FileSegment::reserve(size_t size_to_reserve) return reserved; } -bool FileSegment::isDownloaded() const -{ - std::lock_guard segment_lock(mutex); - return isDownloadedUnlocked(segment_lock); -} - -bool FileSegment::isDownloadedUnlocked(std::lock_guard & /* segment_lock */) const -{ - return is_downloaded; -} - -void FileSegment::setDownloaded([[maybe_unused]] std::lock_guard & segment_lock) +void FileSegment::setDownloadedUnlocked([[maybe_unused]] std::unique_lock & segment_lock) { if (is_downloaded) return; - downloader_id.clear(); + setDownloadState(State::DOWNLOADED); + is_downloaded = true; if (cache_writer) { @@ -424,10 +451,12 @@ void FileSegment::setDownloaded([[maybe_unused]] std::lock_guard & s assert(std::filesystem::file_size(getPathInLocalCache()) > 0); } -void FileSegment::setDownloadFailed(std::lock_guard & /* segment_lock */) +void FileSegment::setDownloadFailedUnlocked(std::unique_lock & segment_lock) { - download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; - downloader_id.clear(); + LOG_INFO(log, "Settings download as failed: {}", getInfoForLogUnlocked(segment_lock)); + + setDownloadState(State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); + resetDownloaderUnlocked(segment_lock); if (cache_writer) { @@ -437,43 +466,31 @@ void FileSegment::setDownloadFailed(std::lock_guard & /* segment_loc } } -void FileSegment::completeBatchAndResetDownloader() +void FileSegment::completePartAndResetDownloader() { - std::lock_guard segment_lock(mutex); + std::unique_lock segment_lock(mutex); + completePartAndResetDownloaderUnlocked(segment_lock); +} - assertNotDetached(segment_lock); +void FileSegment::completePartAndResetDownloaderUnlocked(std::unique_lock & segment_lock) +{ + assertNotDetachedUnlocked(segment_lock); + assertIsDownloaderUnlocked("completePartAndResetDownloader", segment_lock); - if (!isDownloaderImpl(segment_lock)) - { - cv.notify_all(); - throw Exception( - ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "File segment can be completed only by downloader ({} != {})", - downloader_id, getCallerId()); - } - - resetDownloaderImpl(segment_lock); - - LOG_TEST(log, "Complete batch. Current downloaded size: {}", getDownloadedSizeUnlocked(segment_lock)); + resetDownloadingStateUnlocked(segment_lock); + resetDownloaderUnlocked(segment_lock); + LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(segment_lock)); cv.notify_all(); } void FileSegment::completeWithState(State state) { std::lock_guard cache_lock(cache->mutex); - std::lock_guard segment_lock(mutex); + std::unique_lock segment_lock(mutex); - assertNotDetached(segment_lock); - - auto caller_id = getCallerId(); - if (caller_id != downloader_id) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "File segment completion can be done only by downloader. (CallerId: {}, downloader id: {}", - caller_id, downloader_id); - } + assertNotDetachedUnlocked(segment_lock); + assertIsDownloaderUnlocked("complete", segment_lock); if (state != State::DOWNLOADED && state != State::PARTIALLY_DOWNLOADED @@ -485,24 +502,29 @@ void FileSegment::completeWithState(State state) "Cannot complete file segment with state: {}", stateToString(state)); } - download_state = state; + setDownloadState(state); completeBasedOnCurrentState(cache_lock, segment_lock); } -void FileSegment::completeWithoutState(std::lock_guard & cache_lock) +void FileSegment::completeWithoutState() { - std::lock_guard segment_lock(mutex); + std::lock_guard cache_lock(cache->mutex); + completeWithoutStateUnlocked(cache_lock); +} + +void FileSegment::completeWithoutStateUnlocked(std::lock_guard & cache_lock) +{ + std::unique_lock segment_lock(mutex); completeBasedOnCurrentState(cache_lock, segment_lock); } -void FileSegment::completeBasedOnCurrentState(std::lock_guard & cache_lock, std::lock_guard & segment_lock) +void FileSegment::completeBasedOnCurrentState(std::lock_guard & cache_lock, std::unique_lock & segment_lock) { if (is_detached) return; - bool is_downloader = isDownloaderImpl(segment_lock); + bool is_downloader = isDownloaderUnlocked(segment_lock); bool is_last_holder = cache->isLastFileSegmentHolder(key(), offset(), cache_lock, segment_lock); - bool can_update_segment_state = is_downloader || is_last_holder; size_t current_downloaded_size = getDownloadedSizeUnlocked(segment_lock); SCOPE_EXIT({ @@ -512,16 +534,16 @@ void FileSegment::completeBasedOnCurrentState(std::lock_guard & cach } }); - LOG_TEST(log, "Complete without state (is_last_holder: {}). File segment info: {}", is_last_holder, getInfoForLogImpl(segment_lock)); + LOG_TEST( + log, + "Complete based on current state (is_last_holder: {}, {})", + is_last_holder, getInfoForLogUnlocked(segment_lock)); - if (can_update_segment_state) + if (is_downloader) { - if (current_downloaded_size == range().size()) - setDownloaded(segment_lock); - else - download_state = State::PARTIALLY_DOWNLOADED; - - resetDownloaderImpl(segment_lock); + if (download_state == State::DOWNLOADING) /// != in case of completeWithState + resetDownloadingStateUnlocked(segment_lock); + resetDownloaderUnlocked(segment_lock); } switch (download_state) @@ -535,16 +557,17 @@ void FileSegment::completeBasedOnCurrentState(std::lock_guard & cach } case State::DOWNLOADED: { - assert(getDownloadedSizeUnlocked(segment_lock) == range().size()); - assert(isDownloadedUnlocked(segment_lock)); + chassert(getDownloadedSizeUnlocked(segment_lock) == range().size()); + assert(is_downloaded); + assert(!cache_writer); break; } case State::DOWNLOADING: - case State::EMPTY: { - assert(!is_last_holder); + chassert(!is_last_holder); break; } + case State::EMPTY: case State::PARTIALLY_DOWNLOADED: case State::PARTIALLY_DOWNLOADED_NO_CONTINUATION: { @@ -554,7 +577,7 @@ void FileSegment::completeBasedOnCurrentState(std::lock_guard & cach { LOG_TEST(log, "Remove cell {} (nothing downloaded)", range().toString()); - download_state = State::SKIP_CACHE; + setDownloadState(State::SKIP_CACHE); cache->remove(key(), offset(), cache_lock, segment_lock); } else @@ -567,7 +590,7 @@ void FileSegment::completeBasedOnCurrentState(std::lock_guard & cach * in FileSegmentsHolder represent a contiguous range, so we can resize * it only when nobody needs it. */ - download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; + setDownloadState(State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); /// Resize this file segment by creating a copy file segment with DOWNLOADED state, /// but current file segment should remain PARRTIALLY_DOWNLOADED_NO_CONTINUATION and with detached state, @@ -576,23 +599,22 @@ void FileSegment::completeBasedOnCurrentState(std::lock_guard & cach cache->reduceSizeToDownloaded(key(), offset(), cache_lock, segment_lock); } - markAsDetached(segment_lock); + detachAssumeStateFinalized(segment_lock); } break; } } - LOG_TEST(log, "Completed file segment: {}", getInfoForLogImpl(segment_lock)); - assertCorrectnessImpl(segment_lock); + LOG_TEST(log, "Completed file segment: {}", getInfoForLogUnlocked(segment_lock)); } String FileSegment::getInfoForLog() const { - std::lock_guard segment_lock(mutex); - return getInfoForLogImpl(segment_lock); + std::unique_lock segment_lock(mutex); + return getInfoForLogUnlocked(segment_lock); } -String FileSegment::getInfoForLogImpl(std::lock_guard & segment_lock) const +String FileSegment::getInfoForLogUnlocked(std::unique_lock & segment_lock) const { WriteBufferFromOwnString info; info << "File segment: " << range().toString() << ", "; @@ -601,15 +623,18 @@ String FileSegment::getInfoForLogImpl(std::lock_guard & segment_lock info << "downloaded size: " << getDownloadedSizeUnlocked(segment_lock) << ", "; info << "reserved size: " << reserved_size << ", "; info << "downloader id: " << (downloader_id.empty() ? "None" : downloader_id) << ", "; + info << "current write offset: " << getCurrentWriteOffsetUnlocked(segment_lock) << ", "; + info << "first non-downloaded offset: " << getFirstNonDownloadedOffsetUnlocked(segment_lock) << ", "; info << "caller id: " << getCallerId() << ", "; + info << "detached: " << is_detached << ", "; info << "persistent: " << is_persistent; return info.str(); } -void FileSegment::wrapWithCacheInfo(Exception & e, const String & message, std::lock_guard & segment_lock) const +void FileSegment::wrapWithCacheInfo(Exception & e, const String & message, std::unique_lock & segment_lock) const { - e.addMessage(fmt::format("{}, current cache state: {}", message, getInfoForLogImpl(segment_lock))); + e.addMessage(fmt::format("{}, current cache state: {}", message, getInfoForLogUnlocked(segment_lock))); } String FileSegment::stateToString(FileSegment::State state) @@ -634,63 +659,64 @@ String FileSegment::stateToString(FileSegment::State state) void FileSegment::assertCorrectness() const { - std::lock_guard segment_lock(mutex); - assertCorrectnessImpl(segment_lock); + std::unique_lock segment_lock(mutex); + assertCorrectnessUnlocked(segment_lock); } -void FileSegment::assertCorrectnessImpl(std::lock_guard & /* segment_lock */) const +void FileSegment::assertCorrectnessUnlocked(std::unique_lock & segment_lock) const { - assert(downloader_id.empty() == (download_state != FileSegment::State::DOWNLOADING)); - assert(!downloader_id.empty() == (download_state == FileSegment::State::DOWNLOADING)); - assert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(getPathInLocalCache()) > 0); + auto current_downloader = getDownloaderUnlocked(segment_lock); + chassert(current_downloader.empty() == (download_state != FileSegment::State::DOWNLOADING)); + chassert(!current_downloader.empty() == (download_state == FileSegment::State::DOWNLOADING)); + chassert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(getPathInLocalCache()) > 0); } -void FileSegment::throwIfDetached() const -{ - std::lock_guard segment_lock(mutex); - throwIfDetachedUnlocked(segment_lock); -} - -void FileSegment::throwIfDetachedUnlocked(std::lock_guard & segment_lock) const +void FileSegment::throwIfDetachedUnlocked(std::unique_lock & segment_lock) const { throw Exception( ErrorCodes::LOGICAL_ERROR, "Cache file segment is in detached state, operation not allowed. " "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE. " - "Please, retry. File segment info: {}", getInfoForLogImpl(segment_lock)); + "Please, retry. File segment info: {}", getInfoForLogUnlocked(segment_lock)); } +void FileSegment::assertNotDetached() const +{ + std::unique_lock segment_lock(mutex); + assertNotDetachedUnlocked(segment_lock); +} -void FileSegment::assertNotDetached(std::lock_guard & segment_lock) const +void FileSegment::assertNotDetachedUnlocked(std::unique_lock & segment_lock) const { if (is_detached) throwIfDetachedUnlocked(segment_lock); } -void FileSegment::assertDetachedStatus(std::lock_guard & segment_lock) const +void FileSegment::assertDetachedStatus(std::unique_lock & segment_lock) const { /// Detached file segment is allowed to have only a certain subset of states. /// It should be either EMPTY or one of the finalized states. - if (download_state != State::EMPTY && !hasFinalizedState()) + if (download_state != State::EMPTY && !hasFinalizedStateUnlocked(segment_lock)) { throw Exception( ErrorCodes::LOGICAL_ERROR, "Detached file segment has incorrect state: {}", - getInfoForLogImpl(segment_lock)); + getInfoForLogUnlocked(segment_lock)); } } FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard & /* cache_lock */) { - std::lock_guard segment_lock(file_segment->mutex); + std::unique_lock segment_lock(file_segment->mutex); auto snapshot = std::make_shared( file_segment->offset(), file_segment->range().size(), file_segment->key(), nullptr, - State::EMPTY); + State::EMPTY, + CreateFileSegmentSettings{}); snapshot->hits_count = file_segment->getHitsCount(); snapshot->ref_count = file_segment.use_count(); @@ -701,41 +727,43 @@ FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment, std return snapshot; } -bool FileSegment::hasFinalizedState() const +bool FileSegment::hasFinalizedStateUnlocked(std::unique_lock & /* segment_lock */) const { return download_state == State::DOWNLOADED || download_state == State::PARTIALLY_DOWNLOADED_NO_CONTINUATION || download_state == State::SKIP_CACHE; } -void FileSegment::detach( - std::lock_guard & /* cache_lock */, - std::lock_guard & segment_lock) +bool FileSegment::isDetached() const +{ + std::unique_lock segment_lock(mutex); + return is_detached; +} + +void FileSegment::detach(std::lock_guard & /* cache_lock */, std::unique_lock & segment_lock) { - /// Now detached status can be in 2 cases, which do not do any complex logic: - /// 1. there is only 1 remaining file segment holder - /// && it does not need this segment anymore - /// && this file segment was in cache and needs to be removed - /// 2. in read_from_cache_if_exists_otherwise_bypass_cache case if (is_detached) return; - markAsDetached(segment_lock); - download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; - downloader_id.clear(); + if (download_state == State::DOWNLOADING) + resetDownloadingStateUnlocked(segment_lock); + else + setDownloadState(State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); - LOG_DEBUG(log, "Detached file segment: {}", getInfoForLogImpl(segment_lock)); + resetDownloaderUnlocked(segment_lock); + detachAssumeStateFinalized(segment_lock); } -void FileSegment::markAsDetached(std::lock_guard & /* segment_lock */) +void FileSegment::detachAssumeStateFinalized(std::unique_lock & segment_lock) { is_detached = true; CurrentMetrics::add(CurrentMetrics::CacheDetachedFileSegments); + LOG_TEST(log, "Detached file segment: {}", getInfoForLogUnlocked(segment_lock)); } FileSegment::~FileSegment() { - std::lock_guard segment_lock(mutex); + std::unique_lock segment_lock(mutex); if (is_detached) CurrentMetrics::sub(CurrentMetrics::CacheDetachedFileSegments); } @@ -761,7 +789,7 @@ FileSegmentsHolder::~FileSegmentsHolder() bool is_detached = false; { - std::lock_guard segment_lock(file_segment->mutex); + std::unique_lock segment_lock(file_segment->mutex); is_detached = file_segment->isDetached(segment_lock); if (is_detached) file_segment->assertDetachedStatus(segment_lock); @@ -779,7 +807,7 @@ FileSegmentsHolder::~FileSegmentsHolder() /// under the same mutex, because complete() checks for segment pointers. std::lock_guard cache_lock(cache->mutex); - file_segment->completeWithoutState(cache_lock); + file_segment->completeWithoutStateUnlocked(cache_lock); file_segment_it = file_segments.erase(current_file_segment_it); } @@ -802,166 +830,4 @@ String FileSegmentsHolder::toString() return ranges; } -FileSegmentRangeWriter::FileSegmentRangeWriter( - FileCache * cache_, - const FileSegment::Key & key_, - OnCompleteFileSegmentCallback && on_complete_file_segment_func_) - : cache(cache_) - , key(key_) - , current_file_segment_it(file_segments_holder.file_segments.end()) - , on_complete_file_segment_func(on_complete_file_segment_func_) -{ -} - -FileSegments::iterator FileSegmentRangeWriter::allocateFileSegment(size_t offset, bool is_persistent) -{ - /** - * Allocate a new file segment starting `offset`. - * File segment capacity will equal `max_file_segment_size`, but actual size is 0. - */ - - std::lock_guard cache_lock(cache->mutex); - - /// We set max_file_segment_size to be downloaded, - /// if we have less size to write, file segment will be resized in complete() method. - auto file_segment = cache->createFileSegmentForDownload( - key, offset, cache->max_file_segment_size, is_persistent, cache_lock); - return file_segments_holder.add(std::move(file_segment)); -} - -void FileSegmentRangeWriter::completeFileSegment(FileSegment & file_segment) -{ - /** - * Complete file segment based on downaloaded size. - */ - - /// File segment can be detached if space reservation failed. - if (file_segment.isDetached()) - return; - - size_t current_downloaded_size = file_segment.getDownloadedSize(); - - /// file_segment->complete(DOWNLOADED) is not enough, because file segment capacity - /// was initially set with a margin as `max_file_segment_size`. => We need to always - /// resize to actual size after download finished. - if (current_downloaded_size != file_segment.range().size()) - { - /// Current file segment is downloaded as a part of write-through cache - /// and therefore cannot be concurrently accessed. Nevertheless, it can be - /// accessed by cache system tables if someone read from them, - /// therefore we need a mutex. - std::lock_guard segment_lock(file_segment.mutex); - - assert(current_downloaded_size <= file_segment.range().size()); - file_segment.segment_range = FileSegment::Range( - file_segment.segment_range.left, - file_segment.segment_range.left + current_downloaded_size - 1); - file_segment.reserved_size = current_downloaded_size; - } - - { - std::lock_guard cache_lock(cache->mutex); - file_segment.completeWithoutState(cache_lock); - } - - on_complete_file_segment_func(file_segment); -} - -bool FileSegmentRangeWriter::write(const char * data, size_t size, size_t offset, bool is_persistent) -{ - /** - * Write a range of file segments. Allocate file segment of `max_file_segment_size` and write to - * it until it is full and then allocate next file segment. - */ - - if (finalized) - return false; - - auto & file_segments = file_segments_holder.file_segments; - - if (current_file_segment_it == file_segments.end()) - { - current_file_segment_it = allocateFileSegment(current_file_segment_write_offset, is_persistent); - } - else - { - if (current_file_segment_write_offset != offset) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cannot write file segment at offset {}, because current write offset is: {}", - offset, current_file_segment_write_offset); - } - - auto current_file_segment = *current_file_segment_it; - if (current_file_segment->getRemainingSizeToDownload() == 0) - { - completeFileSegment(*current_file_segment); - current_file_segment_it = allocateFileSegment(current_file_segment_write_offset, is_persistent); - } - else if (current_file_segment->getDownloadOffset() != offset) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cannot file segment download offset {} does not match current write offset {}", - current_file_segment->getDownloadOffset(), offset); - } - } - - auto & file_segment = *current_file_segment_it; - - auto downloader = file_segment->getOrSetDownloader(); - if (downloader != FileSegment::getCallerId()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to set a downloader. ({})", file_segment->getInfoForLog()); - - SCOPE_EXIT({ - file_segment->resetDownloader(); - }); - - bool reserved = file_segment->reserve(size); - if (!reserved) - { - file_segment->completeWithState(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); - on_complete_file_segment_func(*file_segment); - - LOG_DEBUG( - &Poco::Logger::get("FileSegmentRangeWriter"), - "Unsuccessful space reservation attempt (size: {}, file segment info: {}", - size, file_segment->getInfoForLog()); - - return false; - } - - (*current_file_segment_it)->write(data, size, offset); - current_file_segment_write_offset += size; - - return true; -} - -void FileSegmentRangeWriter::finalize() -{ - if (finalized) - return; - - auto & file_segments = file_segments_holder.file_segments; - if (file_segments.empty() || current_file_segment_it == file_segments.end()) - return; - - completeFileSegment(**current_file_segment_it); - finalized = true; -} - -FileSegmentRangeWriter::~FileSegmentRangeWriter() -{ - try - { - if (!finalized) - finalize(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } -} - } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index f3fb367792a..617e7173c2f 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -1,11 +1,15 @@ #pragma once -#include +#include +#include #include #include +#include +#include +#include #include -#include +#include namespace Poco { class Logger; } @@ -26,17 +30,25 @@ using FileSegmentPtr = std::shared_ptr; using FileSegments = std::list; -class FileSegment : boost::noncopyable +struct CreateFileSegmentSettings +{ + bool is_persistent = false; +}; + +class FileSegment : private boost::noncopyable, public std::enable_shared_from_this { friend class FileCache; friend struct FileSegmentsHolder; friend class FileSegmentRangeWriter; +friend class StorageSystemFilesystemCache; public: using Key = FileCacheKey; using RemoteFileReaderPtr = std::shared_ptr; using LocalCacheWriterPtr = std::unique_ptr; + using Downloader = std::string; + using DownloaderId = std::string; enum class State { @@ -78,7 +90,7 @@ public: const Key & key_, FileCache * cache_, State download_state_, - bool is_persistent_ = false); + const CreateFileSegmentSettings & create_settings); ~FileSegment(); @@ -101,6 +113,14 @@ public: String toString() const { return fmt::format("[{}, {}]", std::to_string(left), std::to_string(right)); } }; + static String getCallerId(); + + String getInfoForLog() const; + + /** + * ========== Methods to get file segment's constant state ================== + */ + const Range & range() const { return segment_range; } const Key & key() const { return file_key; } @@ -109,11 +129,85 @@ public: bool isPersistent() const { return is_persistent; } + using UniqueId = std::pair; + UniqueId getUniqueId() const { return std::pair(key(), offset()); } + + String getPathInLocalCache() const; + + /** + * ========== Methods for _any_ file segment's owner ======================== + */ + + String getOrSetDownloader(); + + bool isDownloader() const; + + DownloaderId getDownloader() const; + + /// Wait for the change of state from DOWNLOADING to any other. State wait(); - bool reserve(size_t size); + bool isDownloaded() const; - void write(const char * from, size_t size, size_t offset_); + size_t getHitsCount() const { return hits_count; } + + size_t getRefCount() const { return ref_count; } + + void incrementHitsCount() { ++hits_count; } + + size_t getCurrentWriteOffset() const; + + size_t getFirstNonDownloadedOffset() const; + + size_t getDownloadedSize() const; + + /// Now detached status can be used in the following cases: + /// 1. there is only 1 remaining file segment holder + /// && it does not need this segment anymore + /// && this file segment was in cache and needs to be removed + /// 2. in read_from_cache_if_exists_otherwise_bypass_cache case to create NOOP file segments. + /// 3. removeIfExists - method which removes file segments from cache even though + /// it might be used at the moment. + + /// If file segment is detached it means the following: + /// 1. It is not present in FileCache, e.g. will not be visible to any cache user apart from + /// those who acquired shared pointer to this file segment before it was detached. + /// 2. Detached file segment can still be hold by some cache users, but it's state became + /// immutable at the point it was detached, any non-const / stateful method will throw an + /// exception. + void detach(std::lock_guard & cache_lock, std::unique_lock & segment_lock); + + static FileSegmentPtr getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard & cache_lock); + + bool isDetached() const; + + void assertCorrectness() const; + + /** + * ========== Methods for _only_ file segment's `writer` ====================== + */ + + void synchronousWrite(const char * from, size_t size, size_t offset); + + /** + * ========== Methods for _only_ file segment's `downloader` ================== + */ + + /// Try to reserve exactly `size` bytes. + bool reserve(size_t size_to_reserve); + + /// Write data into reserved space. + void write(const char * from, size_t size, size_t offset); + + /// Complete file segment with a certain state. + void completeWithState(State state); + + void completeWithoutState(); + + /// Complete file segment's part which was last written. + void completePartAndResetDownloader(); + + void resetDownloader(); RemoteFileReaderPtr getRemoteFileReader(); @@ -123,91 +217,53 @@ public: void resetRemoteFileReader(); - String getOrSetDownloader(); - - String getDownloader() const; - - void resetDownloader(); - - bool isDownloader() const; - - bool isDownloaded() const; - - static String getCallerId(); - - size_t getDownloadOffset() const; - - size_t getDownloadedSize() const; - - size_t getRemainingSizeToDownload() const; - - void completeBatchAndResetDownloader(); - - void completeWithState(State state); - - String getInfoForLog() const; - - size_t getHitsCount() const { return hits_count; } - - size_t getRefCount() const { return ref_count; } - - void incrementHitsCount() { ++hits_count; } - - void assertCorrectness() const; - - static FileSegmentPtr getSnapshot( - const FileSegmentPtr & file_segment, - std::lock_guard & cache_lock); - - void detach( - std::lock_guard & cache_lock, - std::lock_guard & segment_lock); - - [[noreturn]] void throwIfDetached() const; - - bool isDetached() const; - - String getPathInLocalCache() const; - private: - size_t availableSize() const { return reserved_size - downloaded_size; } + size_t getFirstNonDownloadedOffsetUnlocked(std::unique_lock & segment_lock) const; + size_t getCurrentWriteOffsetUnlocked(std::unique_lock & segment_lock) const; + size_t getDownloadedSizeUnlocked(std::unique_lock & segment_lock) const; - size_t getDownloadedSizeUnlocked(std::lock_guard & segment_lock) const; - String getInfoForLogImpl(std::lock_guard & segment_lock) const; - void assertCorrectnessImpl(std::lock_guard & segment_lock) const; - bool hasFinalizedState() const; + String getInfoForLogUnlocked(std::unique_lock & segment_lock) const; - bool isDetached(std::lock_guard & /* segment_lock */) const { return is_detached; } - void markAsDetached(std::lock_guard & segment_lock); - [[noreturn]] void throwIfDetachedUnlocked(std::lock_guard & segment_lock) const; + String getDownloaderUnlocked(std::unique_lock & segment_lock) const; + void resetDownloaderUnlocked(std::unique_lock & segment_lock); + void resetDownloadingStateUnlocked(std::unique_lock & segment_lock); - void assertDetachedStatus(std::lock_guard & segment_lock) const; - void assertNotDetached(std::lock_guard & segment_lock) const; + void setDownloadState(State state); - void setDownloaded(std::lock_guard & segment_lock); - void setDownloadFailed(std::lock_guard & segment_lock); - bool isDownloaderImpl(std::lock_guard & segment_lock) const; + void setDownloadedUnlocked(std::unique_lock & segment_lock); + void setDownloadFailedUnlocked(std::unique_lock & segment_lock); - bool isDownloadedUnlocked(std::lock_guard & segment_lock) const; + bool hasFinalizedStateUnlocked(std::unique_lock & segment_lock) const; - void wrapWithCacheInfo(Exception & e, const String & message, std::lock_guard & segment_lock) const; + bool isDownloaderUnlocked(std::unique_lock & segment_lock) const; - bool lastFileSegmentHolder() const; + bool isDetached(std::unique_lock & /* segment_lock */) const { return is_detached; } + void detachAssumeStateFinalized(std::unique_lock & segment_lock); + [[noreturn]] void throwIfDetachedUnlocked(std::unique_lock & segment_lock) const; + + void assertDetachedStatus(std::unique_lock & segment_lock) const; + void assertNotDetached() const; + void assertNotDetachedUnlocked(std::unique_lock & segment_lock) const; + void assertIsDownloaderUnlocked(const std::string & operation, std::unique_lock & segment_lock) const; + void assertCorrectnessUnlocked(std::unique_lock & segment_lock) const; /// complete() without any completion state is called from destructor of /// FileSegmentsHolder. complete() might check if the caller of the method /// is the last alive holder of the segment. Therefore, complete() and destruction /// of the file segment pointer must be done under the same cache mutex. - void completeBasedOnCurrentState(std::lock_guard & cache_lock, std::lock_guard & segment_lock); - void completeWithoutState(std::lock_guard & cache_lock); + void completeWithoutStateUnlocked(std::lock_guard & cache_lock); + void completeBasedOnCurrentState(std::lock_guard & cache_lock, std::unique_lock & segment_lock); - void resetDownloaderImpl(std::lock_guard & segment_lock); + void completePartAndResetDownloaderUnlocked(std::unique_lock & segment_lock); + + void wrapWithCacheInfo(Exception & e, const String & message, std::unique_lock & segment_lock) const; Range segment_range; State download_state; - String downloader_id; + /// The one who prepares the download + DownloaderId downloader_id; RemoteFileReaderPtr remote_file_reader; LocalCacheWriterPtr cache_writer; @@ -245,6 +301,7 @@ private: std::atomic ref_count = 0; /// Used for getting snapshot state bool is_persistent; + CurrentMetrics::Increment metric_increment{CurrentMetrics::CacheFileSegments}; }; @@ -268,47 +325,4 @@ struct FileSegmentsHolder : private boost::noncopyable FileSegments file_segments{}; }; -/** - * We want to write eventually some size, which is not known until the very end. - * Therefore we allocate file segments lazily. Each file segment is assigned capacity - * of max_file_segment_size, but reserved_size remains 0, until call to tryReserve(). - * Once current file segment is full (reached max_file_segment_size), we allocate a - * new file segment. All allocated file segments resize in file segments holder. - * If at the end of all writes, the last file segment is not full, then it is resized. - */ -class FileSegmentRangeWriter -{ -public: - using OnCompleteFileSegmentCallback = std::function; - - FileSegmentRangeWriter( - FileCache * cache_, - const FileSegment::Key & key_, - /// A callback which is called right after each file segment is completed. - /// It is used to write into filesystem cache log. - OnCompleteFileSegmentCallback && on_complete_file_segment_func_); - - ~FileSegmentRangeWriter(); - - bool write(const char * data, size_t size, size_t offset, bool is_persistent); - - void finalize(); - -private: - FileSegments::iterator allocateFileSegment(size_t offset, bool is_persistent); - void completeFileSegment(FileSegment & file_segment); - - FileCache * cache; - FileSegment::Key key; - - FileSegmentsHolder file_segments_holder; - FileSegments::iterator current_file_segment_it; - - size_t current_file_segment_write_offset = 0; - - bool finalized = false; - - OnCompleteFileSegmentCallback on_complete_file_segment_func; -}; - } diff --git a/src/Interpreters/CatBoostModel.cpp b/src/Interpreters/CatBoostModel.cpp deleted file mode 100644 index d5803ed9e36..00000000000 --- a/src/Interpreters/CatBoostModel.cpp +++ /dev/null @@ -1,525 +0,0 @@ -#include "CatBoostModel.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ -extern const int LOGICAL_ERROR; -extern const int BAD_ARGUMENTS; -extern const int CANNOT_LOAD_CATBOOST_MODEL; -extern const int CANNOT_APPLY_CATBOOST_MODEL; -} - -/// CatBoost wrapper interface functions. -class CatBoostWrapperAPI -{ -public: - using ModelCalcerHandle = void; - - ModelCalcerHandle * (* ModelCalcerCreate)(); // NOLINT - - void (* ModelCalcerDelete)(ModelCalcerHandle * calcer); // NOLINT - - const char * (* GetErrorString)(); // NOLINT - - bool (* LoadFullModelFromFile)(ModelCalcerHandle * calcer, const char * filename); // NOLINT - - bool (* CalcModelPredictionFlat)(ModelCalcerHandle * calcer, size_t docCount, // NOLINT - const float ** floatFeatures, size_t floatFeaturesSize, - double * result, size_t resultSize); - - bool (* CalcModelPrediction)(ModelCalcerHandle * calcer, size_t docCount, // NOLINT - const float ** floatFeatures, size_t floatFeaturesSize, - const char *** catFeatures, size_t catFeaturesSize, - double * result, size_t resultSize); - - bool (* CalcModelPredictionWithHashedCatFeatures)(ModelCalcerHandle * calcer, size_t docCount, // NOLINT - const float ** floatFeatures, size_t floatFeaturesSize, - const int ** catFeatures, size_t catFeaturesSize, - double * result, size_t resultSize); - - int (* GetStringCatFeatureHash)(const char * data, size_t size); // NOLINT - int (* GetIntegerCatFeatureHash)(uint64_t val); // NOLINT - - size_t (* GetFloatFeaturesCount)(ModelCalcerHandle* calcer); // NOLINT - size_t (* GetCatFeaturesCount)(ModelCalcerHandle* calcer); // NOLINT - size_t (* GetTreeCount)(ModelCalcerHandle* modelHandle); // NOLINT - size_t (* GetDimensionsCount)(ModelCalcerHandle* modelHandle); // NOLINT - - bool (* CheckModelMetadataHasKey)(ModelCalcerHandle* modelHandle, const char* keyPtr, size_t keySize); // NOLINT - size_t (*GetModelInfoValueSize)(ModelCalcerHandle* modelHandle, const char* keyPtr, size_t keySize); // NOLINT - const char* (*GetModelInfoValue)(ModelCalcerHandle* modelHandle, const char* keyPtr, size_t keySize); // NOLINT -}; - - -class CatBoostModelHolder -{ -private: - CatBoostWrapperAPI::ModelCalcerHandle * handle; - const CatBoostWrapperAPI * api; -public: - explicit CatBoostModelHolder(const CatBoostWrapperAPI * api_) : api(api_) { handle = api->ModelCalcerCreate(); } - ~CatBoostModelHolder() { api->ModelCalcerDelete(handle); } - - CatBoostWrapperAPI::ModelCalcerHandle * get() { return handle; } -}; - - -/// Holds CatBoost wrapper library and provides wrapper interface. -class CatBoostLibHolder -{ -public: - explicit CatBoostLibHolder(std::string lib_path_) : lib_path(std::move(lib_path_)), lib(lib_path) { initAPI(); } - - const CatBoostWrapperAPI & getAPI() const { return api; } - const std::string & getCurrentPath() const { return lib_path; } - -private: - CatBoostWrapperAPI api; - std::string lib_path; - SharedLibrary lib; - - void initAPI() - { - load(api.ModelCalcerCreate, "ModelCalcerCreate"); - load(api.ModelCalcerDelete, "ModelCalcerDelete"); - load(api.GetErrorString, "GetErrorString"); - load(api.LoadFullModelFromFile, "LoadFullModelFromFile"); - load(api.CalcModelPredictionFlat, "CalcModelPredictionFlat"); - load(api.CalcModelPrediction, "CalcModelPrediction"); - load(api.CalcModelPredictionWithHashedCatFeatures, "CalcModelPredictionWithHashedCatFeatures"); - load(api.GetStringCatFeatureHash, "GetStringCatFeatureHash"); - load(api.GetIntegerCatFeatureHash, "GetIntegerCatFeatureHash"); - load(api.GetFloatFeaturesCount, "GetFloatFeaturesCount"); - load(api.GetCatFeaturesCount, "GetCatFeaturesCount"); - tryLoad(api.CheckModelMetadataHasKey, "CheckModelMetadataHasKey"); - tryLoad(api.GetModelInfoValueSize, "GetModelInfoValueSize"); - tryLoad(api.GetModelInfoValue, "GetModelInfoValue"); - tryLoad(api.GetTreeCount, "GetTreeCount"); - tryLoad(api.GetDimensionsCount, "GetDimensionsCount"); - } - - template - void load(T& func, const std::string & name) { func = lib.get(name); } - - template - void tryLoad(T& func, const std::string & name) { func = lib.tryGet(name); } -}; - -std::shared_ptr getCatBoostWrapperHolder(const std::string & lib_path) -{ - static std::shared_ptr ptr; - static std::mutex mutex; - - std::lock_guard lock(mutex); - - if (!ptr || ptr->getCurrentPath() != lib_path) - ptr = std::make_shared(lib_path); - - return ptr; -} - -class CatBoostModelImpl -{ -public: - CatBoostModelImpl(const CatBoostWrapperAPI * api_, const std::string & model_path) : api(api_) - { - handle = std::make_unique(api); - if (!handle) - { - throw Exception(ErrorCodes::CANNOT_LOAD_CATBOOST_MODEL, - "Cannot create CatBoost model: {}", - api->GetErrorString()); - } - if (!api->LoadFullModelFromFile(handle->get(), model_path.c_str())) - { - throw Exception(ErrorCodes::CANNOT_LOAD_CATBOOST_MODEL, - "Cannot load CatBoost model: {}", - api->GetErrorString()); - } - - float_features_count = api->GetFloatFeaturesCount(handle->get()); - cat_features_count = api->GetCatFeaturesCount(handle->get()); - tree_count = 1; - if (api->GetDimensionsCount) - tree_count = api->GetDimensionsCount(handle->get()); - } - - ColumnPtr evaluate(const ColumnRawPtrs & columns) const - { - if (columns.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Got empty columns list for CatBoost model."); - - if (columns.size() != float_features_count + cat_features_count) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Number of columns is different with number of features: columns size {} float features size {} + cat features size {}", - columns.size(), - float_features_count, - cat_features_count); - - for (size_t i = 0; i < float_features_count; ++i) - { - if (!columns[i]->isNumeric()) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} should be numeric to make float feature.", i); - } - } - - bool cat_features_are_strings = true; - for (size_t i = float_features_count; i < float_features_count + cat_features_count; ++i) - { - const auto * column = columns[i]; - if (column->isNumeric()) - { - cat_features_are_strings = false; - } - else if (!(typeid_cast(column) - || typeid_cast(column))) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} should be numeric or string.", i); - } - } - - auto result = evalImpl(columns, cat_features_are_strings); - - if (tree_count == 1) - return result; - - size_t column_size = columns.front()->size(); - auto * result_buf = result->getData().data(); - - /// Multiple trees case. Copy data to several columns. - MutableColumns mutable_columns(tree_count); - std::vector column_ptrs(tree_count); - for (size_t i = 0; i < tree_count; ++i) - { - auto col = ColumnFloat64::create(column_size); - column_ptrs[i] = col->getData().data(); - mutable_columns[i] = std::move(col); - } - - Float64 * data = result_buf; - for (size_t row = 0; row < column_size; ++row) - { - for (size_t i = 0; i < tree_count; ++i) - { - *column_ptrs[i] = *data; - ++column_ptrs[i]; - ++data; - } - } - - return ColumnTuple::create(std::move(mutable_columns)); - } - - size_t getFloatFeaturesCount() const { return float_features_count; } - size_t getCatFeaturesCount() const { return cat_features_count; } - size_t getTreeCount() const { return tree_count; } - -private: - std::unique_ptr handle; - const CatBoostWrapperAPI * api; - size_t float_features_count; - size_t cat_features_count; - size_t tree_count; - - /// Buffer should be allocated with features_count * column->size() elements. - /// Place column elements in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] - template - void placeColumnAsNumber(const IColumn * column, T * buffer, size_t features_count) const - { - size_t size = column->size(); - FieldVisitorConvertToNumber visitor; - for (size_t i = 0; i < size; ++i) - { - /// TODO: Replace with column visitor. - Field field; - column->get(i, field); - *buffer = applyVisitor(visitor, field); - buffer += features_count; - } - } - - /// Buffer should be allocated with features_count * column->size() elements. - /// Place string pointers in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] - static void placeStringColumn(const ColumnString & column, const char ** buffer, size_t features_count) - { - size_t size = column.size(); - for (size_t i = 0; i < size; ++i) - { - *buffer = const_cast(column.getDataAtWithTerminatingZero(i).data); - buffer += features_count; - } - } - - /// Buffer should be allocated with features_count * column->size() elements. - /// Place string pointers in positions buffer[0], buffer[features_count], ... , buffer[size * features_count] - /// Returns PODArray which holds data (because ColumnFixedString doesn't store terminating zero). - static PODArray placeFixedStringColumn( - const ColumnFixedString & column, const char ** buffer, size_t features_count) - { - size_t size = column.size(); - size_t str_size = column.getN(); - PODArray data(size * (str_size + 1)); - char * data_ptr = data.data(); - - for (size_t i = 0; i < size; ++i) - { - auto ref = column.getDataAt(i); - memcpy(data_ptr, ref.data, ref.size); - data_ptr[ref.size] = 0; - *buffer = data_ptr; - data_ptr += ref.size + 1; - buffer += features_count; - } - - return data; - } - - /// Place columns into buffer, returns column which holds placed data. Buffer should contains column->size() values. - template - ColumnPtr placeNumericColumns(const ColumnRawPtrs & columns, - size_t offset, size_t size, const T** buffer) const - { - if (size == 0) - return nullptr; - size_t column_size = columns[offset]->size(); - auto data_column = ColumnVector::create(size * column_size); - T * data = data_column->getData().data(); - for (size_t i = 0; i < size; ++i) - { - const auto * column = columns[offset + i]; - if (column->isNumeric()) - placeColumnAsNumber(column, data + i, size); - } - - for (size_t i = 0; i < column_size; ++i) - { - *buffer = data; - ++buffer; - data += size; - } - - return data_column; - } - - /// Place columns into buffer, returns data which was used for fixed string columns. - /// Buffer should contains column->size() values, each value contains size strings. - static std::vector> placeStringColumns( - const ColumnRawPtrs & columns, size_t offset, size_t size, const char ** buffer) - { - if (size == 0) - return {}; - - std::vector> data; - for (size_t i = 0; i < size; ++i) - { - const auto * column = columns[offset + i]; - if (const auto * column_string = typeid_cast(column)) - placeStringColumn(*column_string, buffer + i, size); - else if (const auto * column_fixed_string = typeid_cast(column)) - data.push_back(placeFixedStringColumn(*column_fixed_string, buffer + i, size)); - else - throw Exception("Cannot place string column.", ErrorCodes::LOGICAL_ERROR); - } - - return data; - } - - /// Calc hash for string cat feature at ps positions. - template - void calcStringHashes(const Column * column, size_t ps, const int ** buffer) const - { - size_t column_size = column->size(); - for (size_t j = 0; j < column_size; ++j) - { - auto ref = column->getDataAt(j); - const_cast(*buffer)[ps] = api->GetStringCatFeatureHash(ref.data, ref.size); - ++buffer; - } - } - - /// Calc hash for int cat feature at ps position. Buffer at positions ps should contains unhashed values. - void calcIntHashes(size_t column_size, size_t ps, const int ** buffer) const - { - for (size_t j = 0; j < column_size; ++j) - { - const_cast(*buffer)[ps] = api->GetIntegerCatFeatureHash((*buffer)[ps]); - ++buffer; - } - } - - /// buffer contains column->size() rows and size columns. - /// For int cat features calc hash inplace. - /// For string cat features calc hash from column rows. - void calcHashes(const ColumnRawPtrs & columns, size_t offset, size_t size, const int ** buffer) const - { - if (size == 0) - return; - size_t column_size = columns[offset]->size(); - - std::vector> data; - for (size_t i = 0; i < size; ++i) - { - const auto * column = columns[offset + i]; - if (const auto * column_string = typeid_cast(column)) - calcStringHashes(column_string, i, buffer); - else if (const auto * column_fixed_string = typeid_cast(column)) - calcStringHashes(column_fixed_string, i, buffer); - else - calcIntHashes(column_size, i, buffer); - } - } - - /// buffer[column_size * cat_features_count] -> char * => cat_features[column_size][cat_features_count] -> char * - void fillCatFeaturesBuffer(const char *** cat_features, const char ** buffer, - size_t column_size) const - { - for (size_t i = 0; i < column_size; ++i) - { - *cat_features = buffer; - ++cat_features; - buffer += cat_features_count; - } - } - - /// Convert values to row-oriented format and call evaluation function from CatBoost wrapper api. - /// * CalcModelPredictionFlat if no cat features - /// * CalcModelPrediction if all cat features are strings - /// * CalcModelPredictionWithHashedCatFeatures if has int cat features. - ColumnFloat64::MutablePtr evalImpl( - const ColumnRawPtrs & columns, - bool cat_features_are_strings) const - { - std::string error_msg = "Error occurred while applying CatBoost model: "; - size_t column_size = columns.front()->size(); - - auto result = ColumnFloat64::create(column_size * tree_count); - auto * result_buf = result->getData().data(); - - if (!column_size) - return result; - - /// Prepare float features. - PODArray float_features(column_size); - auto * float_features_buf = float_features.data(); - /// Store all float data into single column. float_features is a list of pointers to it. - auto float_features_col = placeNumericColumns(columns, 0, float_features_count, float_features_buf); - - if (cat_features_count == 0) - { - if (!api->CalcModelPredictionFlat(handle->get(), column_size, - float_features_buf, float_features_count, - result_buf, column_size * tree_count)) - { - - throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); - } - return result; - } - - /// Prepare cat features. - if (cat_features_are_strings) - { - /// cat_features_holder stores pointers to ColumnString data or fixed_strings_data. - PODArray cat_features_holder(cat_features_count * column_size); - PODArray cat_features(column_size); - auto * cat_features_buf = cat_features.data(); - - fillCatFeaturesBuffer(cat_features_buf, cat_features_holder.data(), column_size); - /// Fixed strings are stored without termination zero, so have to copy data into fixed_strings_data. - auto fixed_strings_data = placeStringColumns(columns, float_features_count, - cat_features_count, cat_features_holder.data()); - - if (!api->CalcModelPrediction(handle->get(), column_size, - float_features_buf, float_features_count, - cat_features_buf, cat_features_count, - result_buf, column_size * tree_count)) - { - throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); - } - } - else - { - PODArray cat_features(column_size); - auto * cat_features_buf = cat_features.data(); - auto cat_features_col = placeNumericColumns(columns, float_features_count, - cat_features_count, cat_features_buf); - calcHashes(columns, float_features_count, cat_features_count, cat_features_buf); - if (!api->CalcModelPredictionWithHashedCatFeatures( - handle->get(), column_size, - float_features_buf, float_features_count, - cat_features_buf, cat_features_count, - result_buf, column_size * tree_count)) - { - throw Exception(error_msg + api->GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); - } - } - - return result; - } -}; - -CatBoostModel::CatBoostModel(std::string name_, std::string model_path_, std::string lib_path_, - const ExternalLoadableLifetime & lifetime_) - : name(std::move(name_)), model_path(std::move(model_path_)), lib_path(std::move(lib_path_)), lifetime(lifetime_) -{ - api_provider = getCatBoostWrapperHolder(lib_path); - api = &api_provider->getAPI(); - model = std::make_unique(api, model_path); -} - -CatBoostModel::~CatBoostModel() = default; - -size_t CatBoostModel::getFloatFeaturesCount() const -{ - return model->getFloatFeaturesCount(); -} - -size_t CatBoostModel::getCatFeaturesCount() const -{ - return model->getCatFeaturesCount(); -} - -size_t CatBoostModel::getTreeCount() const -{ - return model->getTreeCount(); -} - -DataTypePtr CatBoostModel::getReturnType() const -{ - size_t tree_count = getTreeCount(); - auto type = std::make_shared(); - if (tree_count == 1) - return type; - - DataTypes types(tree_count, type); - - return std::make_shared(types); -} - -ColumnPtr CatBoostModel::evaluate(const ColumnRawPtrs & columns) const -{ - if (!model) - throw Exception("CatBoost model was not loaded.", ErrorCodes::LOGICAL_ERROR); - - return model->evaluate(columns); -} - -} diff --git a/src/Interpreters/CatBoostModel.h b/src/Interpreters/CatBoostModel.h deleted file mode 100644 index 7bb1df92b67..00000000000 --- a/src/Interpreters/CatBoostModel.h +++ /dev/null @@ -1,73 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ - -class CatBoostLibHolder; -class CatBoostWrapperAPI; -class CatBoostModelImpl; - -class IDataType; -using DataTypePtr = std::shared_ptr; - -/// General ML model evaluator interface. -class IMLModel : public IExternalLoadable -{ -public: - IMLModel() = default; - virtual ColumnPtr evaluate(const ColumnRawPtrs & columns) const = 0; - virtual std::string getTypeName() const = 0; - virtual DataTypePtr getReturnType() const = 0; - virtual ~IMLModel() override = default; -}; - -class CatBoostModel : public IMLModel -{ -public: - CatBoostModel(std::string name, std::string model_path, - std::string lib_path, const ExternalLoadableLifetime & lifetime); - - ~CatBoostModel() override; - - ColumnPtr evaluate(const ColumnRawPtrs & columns) const override; - std::string getTypeName() const override { return "catboost"; } - - size_t getFloatFeaturesCount() const; - size_t getCatFeaturesCount() const; - size_t getTreeCount() const; - DataTypePtr getReturnType() const override; - - /// IExternalLoadable interface. - - const ExternalLoadableLifetime & getLifetime() const override { return lifetime; } - - std::string getLoadableName() const override { return name; } - - bool supportUpdates() const override { return true; } - - bool isModified() const override { return true; } - - std::shared_ptr clone() const override - { - return std::make_shared(name, model_path, lib_path, lifetime); - } - -private: - const std::string name; - std::string model_path; - std::string lib_path; - ExternalLoadableLifetime lifetime; - std::shared_ptr api_provider; - const CatBoostWrapperAPI * api; - - std::unique_ptr model; - - void init(); -}; - -} diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index e67d57628e8..d69878e6af0 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -52,7 +52,6 @@ #include #include #include -#include #include #include #include @@ -143,7 +142,7 @@ namespace ErrorCodes /** Set of known objects (environment), that could be used in query. * Shared (global) part. Order of members (especially, order of destruction) is very important. */ -struct ContextSharedPart +struct ContextSharedPart : boost::noncopyable { Poco::Logger * log = &Poco::Logger::get("Context"); @@ -153,7 +152,6 @@ struct ContextSharedPart mutable std::mutex embedded_dictionaries_mutex; mutable std::mutex external_dictionaries_mutex; mutable std::mutex external_user_defined_executable_functions_mutex; - mutable std::mutex external_models_mutex; /// Separate mutex for storage policies. During server startup we may /// initialize some important storages (system logs with MergeTree engine) /// under context lock. @@ -191,9 +189,7 @@ struct ContextSharedPart mutable std::unique_ptr embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. mutable std::unique_ptr external_dictionaries_loader; mutable std::unique_ptr external_user_defined_executable_functions_loader; - mutable std::unique_ptr external_models_loader; - ExternalLoaderXMLConfigRepository * external_models_config_repository = nullptr; scope_guard models_repository_guard; ExternalLoaderXMLConfigRepository * external_dictionaries_config_repository = nullptr; @@ -215,6 +211,7 @@ struct ContextSharedPart std::unique_ptr access_control; mutable UncompressedCachePtr uncompressed_cache; /// The cache of decompressed blocks. mutable MarkCachePtr mark_cache; /// Cache of marks in compressed files. + mutable std::unique_ptr load_marks_threadpool; /// Threadpool for loading marks cache. mutable UncompressedCachePtr index_uncompressed_cache; /// The cache of decompressed blocks for MergeTree indices. mutable MarkCachePtr index_mark_cache; /// Cache of marks in compressed files of MergeTree indices. mutable MMappedFileCachePtr mmap_cache; /// Cache of mmapped files to avoid frequent open/map/unmap/close and to reuse from several threads. @@ -313,11 +310,19 @@ struct ContextSharedPart ~ContextSharedPart() { - /// Wait for thread pool for background writes, - /// since it may use per-user MemoryTracker which will be destroyed here. try { + /// Wait for thread pool for background writes, + /// since it may use per-user MemoryTracker which will be destroyed here. IObjectStorage::getThreadPoolWriter().wait(); + /// Make sure that threadpool is destructed before this->process_list + /// because thread_status, which was created for threads inside threadpool, + /// relies on it. + if (load_marks_threadpool) + { + load_marks_threadpool->wait(); + load_marks_threadpool.reset(); + } } catch (...) { @@ -350,8 +355,6 @@ struct ContextSharedPart external_dictionaries_loader->enablePeriodicUpdates(false); if (external_user_defined_executable_functions_loader) external_user_defined_executable_functions_loader->enablePeriodicUpdates(false); - if (external_models_loader) - external_models_loader->enablePeriodicUpdates(false); Session::shutdownNamedSessions(); @@ -382,7 +385,6 @@ struct ContextSharedPart std::unique_ptr delete_embedded_dictionaries; std::unique_ptr delete_external_dictionaries_loader; std::unique_ptr delete_external_user_defined_executable_functions_loader; - std::unique_ptr delete_external_models_loader; std::unique_ptr delete_buffer_flush_schedule_pool; std::unique_ptr delete_schedule_pool; std::unique_ptr delete_distributed_schedule_pool; @@ -421,7 +423,6 @@ struct ContextSharedPart delete_embedded_dictionaries = std::move(embedded_dictionaries); delete_external_dictionaries_loader = std::move(external_dictionaries_loader); delete_external_user_defined_executable_functions_loader = std::move(external_user_defined_executable_functions_loader); - delete_external_models_loader = std::move(external_models_loader); delete_buffer_flush_schedule_pool = std::move(buffer_flush_schedule_pool); delete_schedule_pool = std::move(schedule_pool); delete_distributed_schedule_pool = std::move(distributed_schedule_pool); @@ -449,7 +450,6 @@ struct ContextSharedPart delete_embedded_dictionaries.reset(); delete_external_dictionaries_loader.reset(); delete_external_user_defined_executable_functions_loader.reset(); - delete_external_models_loader.reset(); delete_ddl_worker.reset(); delete_buffer_flush_schedule_pool.reset(); delete_schedule_pool.reset(); @@ -1467,48 +1467,6 @@ ExternalUserDefinedExecutableFunctionsLoader & Context::getExternalUserDefinedEx return *shared->external_user_defined_executable_functions_loader; } -const ExternalModelsLoader & Context::getExternalModelsLoader() const -{ - return const_cast(this)->getExternalModelsLoader(); -} - -ExternalModelsLoader & Context::getExternalModelsLoader() -{ - std::lock_guard lock(shared->external_models_mutex); - return getExternalModelsLoaderUnlocked(); -} - -ExternalModelsLoader & Context::getExternalModelsLoaderUnlocked() -{ - if (!shared->external_models_loader) - shared->external_models_loader = - std::make_unique(getGlobalContext()); - return *shared->external_models_loader; -} - -void Context::loadOrReloadModels(const Poco::Util::AbstractConfiguration & config) -{ - auto patterns_values = getMultipleValuesFromConfig(config, "", "models_config"); - std::unordered_set patterns(patterns_values.begin(), patterns_values.end()); - - std::lock_guard lock(shared->external_models_mutex); - - auto & external_models_loader = getExternalModelsLoaderUnlocked(); - - if (shared->external_models_config_repository) - { - shared->external_models_config_repository->updatePatterns(patterns); - external_models_loader.reloadConfig(shared->external_models_config_repository->getName()); - return; - } - - auto app_path = getPath(); - auto config_path = getConfigRef().getString("config-file", "config.xml"); - auto repository = std::make_unique(app_path, config_path, patterns); - shared->external_models_config_repository = repository.get(); - shared->models_repository_guard = external_models_loader.addConfigRepository(std::move(repository)); -} - EmbeddedDictionaries & Context::getEmbeddedDictionariesImpl(const bool throw_on_error) const { std::lock_guard lock(shared->embedded_dictionaries_mutex); @@ -1688,6 +1646,17 @@ void Context::dropMarkCache() const shared->mark_cache->reset(); } +ThreadPool & Context::getLoadMarksThreadpool() const +{ + auto lock = getLock(); + if (!shared->load_marks_threadpool) + { + constexpr size_t pool_size = 50; + constexpr size_t queue_size = 1000000; + shared->load_marks_threadpool = std::make_unique(pool_size, pool_size, queue_size); + } + return *shared->load_marks_threadpool; +} void Context::setIndexUncompressedCache(size_t max_size_in_bytes) { @@ -3429,6 +3398,8 @@ ReadSettings Context::getReadSettings() const res.local_fs_prefetch = settings.local_filesystem_read_prefetch; res.remote_fs_prefetch = settings.remote_filesystem_read_prefetch; + res.load_marks_asynchronously = settings.load_marks_asynchronously; + res.remote_fs_read_max_backoff_ms = settings.remote_fs_read_max_backoff_ms; res.remote_fs_read_backoff_max_tries = settings.remote_fs_read_backoff_max_tries; res.enable_filesystem_cache = settings.enable_filesystem_cache; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 77f7d6cbfdd..67cf584d5a7 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -53,7 +53,6 @@ class AccessRightsElements; enum class RowPolicyFilterType; class EmbeddedDictionaries; class ExternalDictionariesLoader; -class ExternalModelsLoader; class ExternalUserDefinedExecutableFunctionsLoader; class InterserverCredentials; using InterserverCredentialsPtr = std::shared_ptr; @@ -612,6 +611,7 @@ public: void killCurrentQuery(); + bool hasInsertionTable() const { return !insertion_table.empty(); } void setInsertionTable(StorageID db_and_table) { insertion_table = std::move(db_and_table); } const StorageID & getInsertionTable() const { return insertion_table; } @@ -644,19 +644,15 @@ public: const EmbeddedDictionaries & getEmbeddedDictionaries() const; const ExternalDictionariesLoader & getExternalDictionariesLoader() const; - const ExternalModelsLoader & getExternalModelsLoader() const; const ExternalUserDefinedExecutableFunctionsLoader & getExternalUserDefinedExecutableFunctionsLoader() const; EmbeddedDictionaries & getEmbeddedDictionaries(); ExternalDictionariesLoader & getExternalDictionariesLoader(); ExternalDictionariesLoader & getExternalDictionariesLoaderUnlocked(); ExternalUserDefinedExecutableFunctionsLoader & getExternalUserDefinedExecutableFunctionsLoader(); ExternalUserDefinedExecutableFunctionsLoader & getExternalUserDefinedExecutableFunctionsLoaderUnlocked(); - ExternalModelsLoader & getExternalModelsLoader(); - ExternalModelsLoader & getExternalModelsLoaderUnlocked(); void tryCreateEmbeddedDictionaries(const Poco::Util::AbstractConfiguration & config) const; void loadOrReloadDictionaries(const Poco::Util::AbstractConfiguration & config); void loadOrReloadUserDefinedExecutableFunctions(const Poco::Util::AbstractConfiguration & config); - void loadOrReloadModels(const Poco::Util::AbstractConfiguration & config); #if USE_NLP SynonymsExtensions & getSynonymsExtensions() const; @@ -806,6 +802,7 @@ public: void setMarkCache(size_t cache_size_in_bytes, const String & mark_cache_policy); std::shared_ptr getMarkCache() const; void dropMarkCache() const; + ThreadPool & getLoadMarksThreadpool() const; /// Create a cache of index uncompressed blocks of specified size. This can be done only once. void setIndexUncompressedCache(size_t max_size_in_bytes); diff --git a/src/Interpreters/CrossToInnerJoinVisitor.cpp b/src/Interpreters/CrossToInnerJoinVisitor.cpp index 52c457f2373..bafa63e767f 100644 --- a/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -115,7 +115,7 @@ std::map> moveExpressionToJoinOn( const Aliases & aliases) { std::map> asts_to_join_on; - for (const auto & node : collectConjunctions(ast)) + for (const auto & node : splitConjunctionsAst(ast)) { if (const auto * func = node->as(); func && func->name == NameEquals::name) { diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 6ec20ab5f5f..c8878297c02 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -890,7 +890,7 @@ void DDLWorker::cleanupQueue(Int64, const ZooKeeperPtr & zookeeper) /// We recursively delete all nodes except node_path/finished to prevent staled hosts from /// creating node_path/active node (see createStatusDirs(...)) - zookeeper->tryRemoveChildrenRecursive(node_path, /* probably_flat */ false, "finished"); + zookeeper->tryRemoveChildrenRecursive(node_path, /* probably_flat */ false, zkutil::RemoveException{"finished"}); /// And then we remove node_path and node_path/finished in a single transaction Coordination::Requests ops; diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index 7ddcc80c02a..e3c1fa4c271 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -61,18 +61,23 @@ public: return host_fqdn_id; } + std::string getQueueDir() const + { + return queue_dir; + } + void startup(); virtual void shutdown(); bool isCurrentlyActive() const { return initialized && !stop_flag; } -protected: /// Returns cached ZooKeeper session (possibly expired). ZooKeeperPtr tryGetZooKeeper() const; /// If necessary, creates a new session and caches it. ZooKeeperPtr getAndSetZooKeeper(); +protected: /// Iterates through queue tasks in ZooKeeper, runs execution of new tasks void scheduleTasks(bool reinitialized); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index be32125edf8..9daa42bf499 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1988,6 +1988,23 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( } } + // Here we need to set order by expression as required output to avoid + // their removal from the ActionsDAG. + const auto * select_query = query_analyzer.getSelectQuery(); + if (select_query->orderBy()) + { + for (auto & child : select_query->orderBy()->children) + { + auto * ast = child->as(); + ASTPtr order_expression = ast->children.at(0); + if (auto * function = order_expression->as(); + function && (function->is_window_function || function->compute_after_window_functions)) + continue; + const String & column_name = order_expression->getColumnName(); + chain.getLastStep().addRequiredOutput(column_name); + } + } + before_window = chain.getLastActions(); finalize_chain(chain); @@ -2007,7 +2024,6 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( // produced the expressions required to calculate window functions. // They are not needed in the final SELECT result. Knowing the correct // list of columns is important when we apply SELECT DISTINCT later. - const auto * select_query = query_analyzer.getSelectQuery(); for (const auto & child : select_query->select()->children) { step.addRequiredOutput(child->getColumnName()); diff --git a/src/Interpreters/ExternalModelsLoader.cpp b/src/Interpreters/ExternalModelsLoader.cpp deleted file mode 100644 index 317cf0bf1c9..00000000000 --- a/src/Interpreters/ExternalModelsLoader.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int INVALID_CONFIG_PARAMETER; -} - - -ExternalModelsLoader::ExternalModelsLoader(ContextPtr context_) - : ExternalLoader("external model", &Poco::Logger::get("ExternalModelsLoader")), WithContext(context_) -{ - setConfigSettings({"model", "name", {}, {}}); - enablePeriodicUpdates(true); -} - -std::shared_ptr ExternalModelsLoader::create( - const std::string & name, const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, const std::string & /* repository_name */) const -{ - String type = config.getString(config_prefix + ".type"); - ExternalLoadableLifetime lifetime(config, config_prefix + ".lifetime"); - - /// TODO: add models factory. - if (type == "catboost") - { - return std::make_unique( - name, config.getString(config_prefix + ".path"), - getContext()->getConfigRef().getString("catboost_dynamic_library_path"), - lifetime - ); - } - else - { - throw Exception("Unknown model type: " + type, ErrorCodes::INVALID_CONFIG_PARAMETER); - } -} -} diff --git a/src/Interpreters/ExternalModelsLoader.h b/src/Interpreters/ExternalModelsLoader.h deleted file mode 100644 index 0eeb60008c3..00000000000 --- a/src/Interpreters/ExternalModelsLoader.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include - - -namespace DB -{ - -/// Manages user-defined models. -class ExternalModelsLoader : public ExternalLoader, WithContext -{ -public: - using ModelPtr = std::shared_ptr; - - /// Models will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. - explicit ExternalModelsLoader(ContextPtr context_); - - ModelPtr getModel(const std::string & model_name) const - { - return std::static_pointer_cast(load(model_name)); - } - - void reloadModel(const std::string & model_name) const - { - loadOrReload(model_name); - } - -protected: - LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, const std::string & repository_name) const override; - - friend class StorageSystemModels; -}; - -} diff --git a/src/Interpreters/FilesystemCacheLog.cpp b/src/Interpreters/FilesystemCacheLog.cpp index a0c8986a8ca..ea2aa3c6bea 100644 --- a/src/Interpreters/FilesystemCacheLog.cpp +++ b/src/Interpreters/FilesystemCacheLog.cpp @@ -21,7 +21,7 @@ static String typeToString(FilesystemCacheLogElement::CacheType type) case FilesystemCacheLogElement::CacheType::READ_FROM_FS_BYPASSING_CACHE: return "READ_FROM_FS_BYPASSING_CACHE"; case FilesystemCacheLogElement::CacheType::WRITE_THROUGH_CACHE: - return "READ_FROM_FS_BYPASSING_CACHE"; + return "WRITE_THROUGH_CACHE"; } __builtin_unreachable(); } diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index f0658cb7c9b..d3750e98b8c 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -322,22 +322,35 @@ std::optional IdentifierMembershipCollector::getIdentsMembership(ASTPtr return IdentifierSemantic::getIdentsMembership(ast, tables, aliases); } -static void collectConjunctions(const ASTPtr & node, std::vector & members) +void splitConjunctionsAst(const ASTPtr & node, ASTs & result) { - if (const auto * func = node->as(); func && func->name == "and") - { - for (const auto & child : func->arguments->children) - collectConjunctions(child, members); + if (!node) return; + + result.emplace_back(node); + + for (size_t idx = 0; idx < result.size();) + { + ASTPtr expression = result.at(idx); + + if (const auto * function = expression->as(); function && function->name == "and") + { + result.erase(result.begin() + idx); + + for (auto & child : function->arguments->children) + result.emplace_back(child); + + continue; + } + ++idx; } - members.push_back(node); } -std::vector collectConjunctions(const ASTPtr & node) +ASTs splitConjunctionsAst(const ASTPtr & node) { - std::vector members; - collectConjunctions(node, members); - return members; + std::vector result; + splitConjunctionsAst(node, result); + return result; } } diff --git a/src/Interpreters/IdentifierSemantic.h b/src/Interpreters/IdentifierSemantic.h index c082e83b75c..178bd291beb 100644 --- a/src/Interpreters/IdentifierSemantic.h +++ b/src/Interpreters/IdentifierSemantic.h @@ -105,6 +105,7 @@ private: }; /// Split expression `expr_1 AND expr_2 AND ... AND expr_n` into vector `[expr_1, expr_2, ..., expr_n]` -std::vector collectConjunctions(const ASTPtr & node); +ASTs splitConjunctionsAst(const ASTPtr & node); +void splitConjunctionsAst(const ASTPtr & node, ASTs & result); } diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 762afd28923..82f635017c9 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -78,15 +78,11 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) query_ptr->as().setDatabase(table_id.database_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); - if (typeid_cast(database.get()) - && !getContext()->getClientInfo().is_replicated_database_internal - && !alter.isAttachAlter() - && !alter.isFetchAlter() - && !alter.isDropPartitionAlter()) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext()); } StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index 5117b92efdf..714bcd6d356 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -38,11 +38,11 @@ BlockIO InterpreterCreateIndexQuery::execute() query_ptr->as().setDatabase(table_id.database_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); - if (typeid_cast(database.get()) && !current_context->getClientInfo().is_replicated_database_internal) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); - return assert_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, current_context); + return database->tryEnqueueReplicatedDDL(query_ptr, current_context); } StoragePtr table = DatabaseCatalog::instance().getTable(table_id, current_context); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index e66fe543ab0..21a02ba351f 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1001,27 +1001,27 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) String current_database = getContext()->getCurrentDatabase(); auto database_name = create.database ? create.getDatabase() : current_database; + DDLGuardPtr ddl_guard; + // If this is a stub ATTACH query, read the query definition from the database if (create.attach && !create.storage && !create.columns_list) { auto database = DatabaseCatalog::instance().getDatabase(database_name); - - if (database->getEngineName() == "Replicated") + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); - - if (auto * ptr = typeid_cast(database.get()); - ptr && !getContext()->getClientInfo().is_replicated_database_internal) - { - create.setDatabase(database_name); - guard->releaseTableLock(); - return ptr->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); - } + create.setDatabase(database_name); + guard->releaseTableLock(); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); } if (!create.cluster.empty()) return executeQueryOnCluster(create); + /// For short syntax of ATTACH query we have to lock table name here, before reading metadata + /// and hold it until table is attached + ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); + bool if_not_exists = create.if_not_exists; // Table SQL definition is available even if the table is detached (even permanently) @@ -1053,6 +1053,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.attach_from_path) { + chassert(!ddl_guard); fs::path user_files = fs::path(getContext()->getUserFilesPath()).lexically_normal(); fs::path root_path = fs::path(getContext()->getPath()).lexically_normal(); @@ -1145,27 +1146,30 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (need_add_to_database) database = DatabaseCatalog::instance().getDatabase(database_name); - if (need_add_to_database && database->getEngineName() == "Replicated") + if (need_add_to_database && database->shouldReplicateQuery(getContext(), query_ptr)) { + chassert(!ddl_guard); auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); - - if (auto * ptr = typeid_cast(database.get()); - ptr && !getContext()->getClientInfo().is_replicated_database_internal) - { - assertOrSetUUID(create, database); - guard->releaseTableLock(); - return ptr->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); - } + assertOrSetUUID(create, database); + guard->releaseTableLock(); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); } if (!create.cluster.empty()) + { + chassert(!ddl_guard); return executeQueryOnCluster(create); + } if (create.replace_table) + { + chassert(!ddl_guard); return doCreateOrReplaceTable(create, properties); + } /// Actually creates table - bool created = doCreateTable(create, properties); + bool created = doCreateTable(create, properties, ddl_guard); + ddl_guard.reset(); if (!created) /// Table already exists return {}; @@ -1180,7 +1184,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) } bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, - const InterpreterCreateQuery::TableProperties & properties) + const InterpreterCreateQuery::TableProperties & properties, + DDLGuardPtr & ddl_guard) { if (create.temporary) { @@ -1193,16 +1198,12 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, return true; } - std::unique_ptr guard; + if (!ddl_guard) + ddl_guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); String data_path; DatabasePtr database; - /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). - * If table doesn't exist, one thread is creating table, while others wait in DDLGuard. - */ - guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); - database = DatabaseCatalog::instance().getDatabase(create.getDatabase()); assertOrSetUUID(create, database); @@ -1411,7 +1412,9 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, try { /// Create temporary table (random name will be generated) - [[maybe_unused]] bool done = InterpreterCreateQuery(query_ptr, create_context).doCreateTable(create, properties); + DDLGuardPtr ddl_guard; + [[maybe_unused]] bool done = InterpreterCreateQuery(query_ptr, create_context).doCreateTable(create, properties, ddl_guard); + ddl_guard.reset(); assert(done); created = true; diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 984310b2952..4d11387f44c 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -18,7 +18,9 @@ class ASTExpressionList; class ASTConstraintDeclaration; class ASTStorage; class IDatabase; +class DDLGuard; using DatabasePtr = std::shared_ptr; +using DDLGuardPtr = std::unique_ptr; /** Allows to create new table or database, @@ -89,7 +91,7 @@ private: AccessRightsElements getRequiredAccess() const; /// Create IStorage and add it to database. If table already exists and IF NOT EXISTS specified, do nothing and return false. - bool doCreateTable(ASTCreateQuery & create, const TableProperties & properties); + bool doCreateTable(ASTCreateQuery & create, const TableProperties & properties, DDLGuardPtr & ddl_guard); BlockIO doCreateOrReplaceTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties); /// Inserts data in created table if it's CREATE ... SELECT BlockIO fillTableIfNeeded(const ASTCreateQuery & create); diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 51fb6cfb948..b5b8ae81366 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -48,12 +48,11 @@ BlockIO InterpreterDeleteQuery::execute() throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); - if (typeid_cast(database.get()) - && !getContext()->getClientInfo().is_replicated_database_internal) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext()); } auto table_lock = table->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp index 70f35a92688..98d48942487 100644 --- a/src/Interpreters/InterpreterDropIndexQuery.cpp +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -36,11 +36,11 @@ BlockIO InterpreterDropIndexQuery::execute() query_ptr->as().setDatabase(table_id.database_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); - if (typeid_cast(database.get()) && !current_context->getClientInfo().is_replicated_database_internal) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); - return assert_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, current_context); + return database->tryEnqueueReplicatedDDL(query_ptr, current_context); } StoragePtr table = DatabaseCatalog::instance().getTable(table_id, current_context); diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 1f83992dbbe..71d65ee7fed 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -139,9 +139,6 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue /// Prevents recursive drop from drop database query. The original query must specify a table. bool is_drop_or_detach_database = !query_ptr->as()->table; - bool is_replicated_ddl_query = typeid_cast(database.get()) && - !context_->getClientInfo().is_replicated_database_internal && - !is_drop_or_detach_database; AccessFlags drop_storage; @@ -152,7 +149,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue else drop_storage = AccessType::DROP_TABLE; - if (is_replicated_ddl_query) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { if (query.kind == ASTDropQuery::Kind::Detach) context_->checkAccess(drop_storage, table_id); @@ -163,7 +160,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue ddl_guard->releaseTableLock(); table.reset(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query.clone(), context_); + return database->tryEnqueueReplicatedDDL(query.clone(), context_); } if (query.kind == ASTDropQuery::Kind::Detach) diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index c22863ef8e5..666a674b2c8 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -107,7 +107,7 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c } DatabasePtr database = database_catalog.getDatabase(elem.from_database_name); - if (typeid_cast(database.get()) && !getContext()->getClientInfo().is_replicated_database_internal) + if (database->shouldReplicateQuery(getContext(), query_ptr)) { if (1 < descriptions.size()) throw Exception( @@ -120,7 +120,7 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c UniqueTableName to(elem.to_database_name, elem.to_table_name); ddl_guards[from]->releaseTableLock(); ddl_guards[to]->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext()); } else { diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index bac3fa850f2..56e87d6a4fb 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -36,6 +35,8 @@ #include #include #include +#include +#include #include #include #include @@ -387,17 +388,15 @@ BlockIO InterpreterSystemQuery::execute() case Type::RELOAD_MODEL: { getContext()->checkAccess(AccessType::SYSTEM_RELOAD_MODEL); - - auto & external_models_loader = system_context->getExternalModelsLoader(); - external_models_loader.reloadModel(query.target_model); + auto bridge_helper = std::make_unique(getContext(), query.target_model); + bridge_helper->removeModel(); break; } case Type::RELOAD_MODELS: { getContext()->checkAccess(AccessType::SYSTEM_RELOAD_MODEL); - - auto & external_models_loader = system_context->getExternalModelsLoader(); - external_models_loader.reloadAllTriedToLoad(); + auto bridge_helper = std::make_unique(getContext()); + bridge_helper->removeAllModels(); break; } case Type::RELOAD_FUNCTION: @@ -424,6 +423,10 @@ BlockIO InterpreterSystemQuery::execute() getContext()->checkAccess(AccessType::SYSTEM_RELOAD_CONFIG); system_context->reloadConfig(); break; + case Type::RELOAD_USERS: + getContext()->checkAccess(AccessType::SYSTEM_RELOAD_USERS); + system_context->getAccessControl().reload(AccessControl::ReloadMode::ALL); + break; case Type::RELOAD_SYMBOLS: { #if defined(__ELF__) && !defined(OS_FREEBSD) @@ -539,7 +542,7 @@ BlockIO InterpreterSystemQuery::execute() { getContext()->checkAccess(AccessType::SYSTEM_UNFREEZE); /// The result contains information about deleted parts as a table. It is for compatibility with ALTER TABLE UNFREEZE query. - result = Unfreezer().unfreeze(query.backup_name, getContext()); + result = Unfreezer(getContext()).systemUnfreeze(query.backup_name); break; } default: @@ -902,6 +905,11 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() required_access.emplace_back(AccessType::SYSTEM_RELOAD_CONFIG); break; } + case Type::RELOAD_USERS: + { + required_access.emplace_back(AccessType::SYSTEM_RELOAD_USERS); + break; + } case Type::RELOAD_SYMBOLS: { required_access.emplace_back(AccessType::SYSTEM_RELOAD_SYMBOLS); diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp index 353ab84674c..99646084e5a 100644 --- a/src/Interpreters/JIT/compileFunction.cpp +++ b/src/Interpreters/JIT/compileFunction.cpp @@ -739,7 +739,10 @@ CompiledAggregateFunctions compileAggregateFunctions(CHJIT & jit, const std::vec { compileCreateAggregateStatesFunctions(module, functions, create_aggregate_states_functions_name); compileAddIntoAggregateStatesFunctions(module, functions, add_aggregate_states_functions_name); - compileAddIntoAggregateStatesFunctionsSinglePlace(module, functions, add_aggregate_states_functions_name_single_place); + /// FIXME: this leads to use-of-uninitialized-value in llvm + /// But for now, it is safe, since it is not used by Aggregator anyway + (void)compileAddIntoAggregateStatesFunctionsSinglePlace; + /// compileAddIntoAggregateStatesFunctionsSinglePlace(module, functions, add_aggregate_states_functions_name_single_place); compileMergeAggregatesStates(module, functions, merge_aggregate_states_functions_name); compileInsertAggregatesIntoResultColumns(module, functions, insert_aggregate_states_functions_name); }); @@ -752,7 +755,7 @@ CompiledAggregateFunctions compileAggregateFunctions(CHJIT & jit, const std::vec assert(create_aggregate_states_function); assert(add_into_aggregate_states_function); - assert(add_into_aggregate_states_function_single_place); + /// assert(add_into_aggregate_states_function_single_place); /// FIXME assert(merge_aggregate_states_function); assert(insert_aggregate_states_function); diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index c496995ba65..26b8bce1f4a 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -473,6 +473,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) dependencies = getAllColumnDependencies(metadata_snapshot, updated_columns); + std::vector read_columns; /// First, break a sequence of commands into stages. for (auto & command : commands) { @@ -706,17 +707,23 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) else if (command.type == MutationCommand::READ_COLUMN) { mutation_kind.set(MutationKind::MUTATE_OTHER); - if (stages.empty() || !stages.back().column_to_updated.empty()) - stages.emplace_back(context); - if (stages.size() == 1) /// First stage only supports filtering and can't update columns. - stages.emplace_back(context); - - stages.back().column_to_updated.emplace(command.column_name, std::make_shared(command.column_name)); + read_columns.emplace_back(command.column_name); } else throw Exception("Unknown mutation command type: " + DB::toString(command.type), ErrorCodes::UNKNOWN_MUTATION_COMMAND); } + if (!read_columns.empty()) + { + if (stages.empty() || !stages.back().column_to_updated.empty()) + stages.emplace_back(context); + if (stages.size() == 1) /// First stage only supports filtering and can't update columns. + stages.emplace_back(context); + + for (auto & column_name : read_columns) + stages.back().column_to_updated.emplace(column_name, std::make_shared(column_name)); + } + /// We care about affected indices and projections because we also need to rewrite them /// when one of index columns updated or filtered with delete. /// The same about columns, that are needed for calculation of TTL expressions. diff --git a/src/Interpreters/PredicateExpressionsOptimizer.cpp b/src/Interpreters/PredicateExpressionsOptimizer.cpp index 68dc8a0e311..fd77f651ff5 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -51,42 +52,14 @@ bool PredicateExpressionsOptimizer::optimize(ASTSelectQuery & select_query) return false; } -static ASTs splitConjunctionPredicate(const std::initializer_list & predicates) -{ - std::vector res; - - for (const auto & predicate : predicates) - { - if (!predicate) - continue; - - res.emplace_back(predicate); - - for (size_t idx = 0; idx < res.size();) - { - ASTPtr expression = res.at(idx); - - if (const auto * function = expression->as(); function && function->name == "and") - { - res.erase(res.begin() + idx); - - for (auto & child : function->arguments->children) - res.emplace_back(child); - - continue; - } - ++idx; - } - } - - return res; -} - std::vector PredicateExpressionsOptimizer::extractTablesPredicates(const ASTPtr & where, const ASTPtr & prewhere) { std::vector tables_predicates(tables_with_columns.size()); - for (const auto & predicate_expression : splitConjunctionPredicate({where, prewhere})) + ASTs predicate_expressions; + splitConjunctionsAst(where, predicate_expressions); + splitConjunctionsAst(prewhere, predicate_expressions); + for (const auto & predicate_expression : predicate_expressions) { ExpressionInfoVisitor::Data expression_info{WithContext{getContext()}, tables_with_columns}; ExpressionInfoVisitor(expression_info).visit(predicate_expression); @@ -186,7 +159,7 @@ bool PredicateExpressionsOptimizer::tryMovePredicatesFromHavingToWhere(ASTSelect return res; }; - for (const auto & moving_predicate: splitConjunctionPredicate({select_query.having()})) + for (const auto & moving_predicate : splitConjunctionsAst(select_query.having())) { TablesWithColumns tables; ExpressionInfoVisitor::Data expression_info{WithContext{getContext()}, tables}; diff --git a/src/Interpreters/PredicateRewriteVisitor.cpp b/src/Interpreters/PredicateRewriteVisitor.cpp index 910c846d58b..34a728da35b 100644 --- a/src/Interpreters/PredicateRewriteVisitor.cpp +++ b/src/Interpreters/PredicateRewriteVisitor.cpp @@ -133,6 +133,36 @@ static void cleanAliasAndCollectIdentifiers(ASTPtr & predicate, std::vectoras()) + { + for (auto & children : predicate->children) + useAliasInsteadOfIdentifier(children); + } + + if (const auto alias = predicate->tryGetAlias(); !alias.empty()) + { + if (ASTIdentifier * identifier = predicate->as()) + identifier->setShortName(alias); + predicate->setAlias({}); + } +} + +static void getConjunctionHashesFrom(const ASTPtr & ast, std::set & hashes) +{ + for (const auto & pred : splitConjunctionsAst(ast)) + { + /// Clone not to modify `ast` + ASTPtr pred_copy = pred->clone(); + useAliasInsteadOfIdentifier(pred_copy); + hashes.emplace(pred_copy->getTreeHash()); + } +} + bool PredicateRewriteVisitorData::rewriteSubquery(ASTSelectQuery & subquery, const Names & inner_columns) { if ((!optimize_final && subquery.final()) @@ -143,12 +173,27 @@ bool PredicateRewriteVisitorData::rewriteSubquery(ASTSelectQuery & subquery, con return false; Names outer_columns = table_columns.columns.getNames(); + + /// Do not add same conditions twice to avoid extra rewrites with exponential blowup + /// (e.g. in case of deep complex query with lots of JOINs) + std::set hashes; + getConjunctionHashesFrom(subquery.where(), hashes); + getConjunctionHashesFrom(subquery.having(), hashes); + + bool is_changed = false; for (const auto & predicate : predicates) { std::vector identifiers; ASTPtr optimize_predicate = predicate->clone(); cleanAliasAndCollectIdentifiers(optimize_predicate, identifiers); + auto predicate_hash = optimize_predicate->getTreeHash(); + if (hashes.contains(predicate_hash)) + continue; + + hashes.emplace(predicate_hash); + is_changed = true; + for (const auto & identifier : identifiers) { IdentifierSemantic::setColumnShortName(*identifier, table_columns.table); @@ -169,7 +214,7 @@ bool PredicateRewriteVisitorData::rewriteSubquery(ASTSelectQuery & subquery, con subquery.having() ? makeASTFunction("and", optimize_predicate, subquery.having()) : optimize_predicate); } - return true; + return is_changed; } } diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index e21d3814086..e7cf5e14143 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -297,6 +297,9 @@ ProcessListEntry::~ProcessListEntry() } } + /// Wait for the query if it is in the cancellation right now. + parent.cancelled_cv.wait(lock.lock, [&]() { return it->is_cancelling == false; }); + /// This removes the memory_tracker of one request. parent.processes.erase(it); @@ -369,6 +372,12 @@ CancellationCode QueryStatus::cancelQuery(bool) void QueryStatus::addPipelineExecutor(PipelineExecutor * e) { + /// In case of asynchronous distributed queries it is possible to call + /// addPipelineExecutor() from the cancelQuery() context, and this will + /// lead to deadlock. + if (is_killed.load()) + throw Exception("Query was cancelled", ErrorCodes::QUERY_WAS_CANCELLED); + std::lock_guard lock(executors_mutex); assert(std::find(executors.begin(), executors.end(), e) == executors.end()); executors.push_back(e); @@ -430,12 +439,35 @@ QueryStatus * ProcessList::tryGetProcessListElement(const String & current_query CancellationCode ProcessList::sendCancelToQuery(const String & current_query_id, const String & current_user, bool kill) { - auto lock = safeLock(); + QueryStatus * elem; - QueryStatus * elem = tryGetProcessListElement(current_query_id, current_user); + /// Cancelling the query should be done without the lock. + /// + /// Since it may be not that trivial, for example in case of distributed + /// queries it tries to cancel the query gracefully on shards and this can + /// take a while, so acquiring a lock during this time will lead to wait + /// all new queries for this cancellation. + /// + /// Another problem is that it can lead to a deadlock, because of + /// OvercommitTracker. + /// + /// So here we first set is_cancelling, and later reset it. + /// The ProcessListEntry cannot be destroy if is_cancelling is true. + { + auto lock = safeLock(); + elem = tryGetProcessListElement(current_query_id, current_user); + if (!elem) + return CancellationCode::NotFound; + elem->is_cancelling = true; + } - if (!elem) - return CancellationCode::NotFound; + SCOPE_EXIT({ + DENY_ALLOCATIONS_IN_SCOPE; + + auto lock = unsafeLock(); + elem->is_cancelling = false; + cancelled_cv.notify_all(); + }); return elem->cancelQuery(kill); } @@ -443,10 +475,28 @@ CancellationCode ProcessList::sendCancelToQuery(const String & current_query_id, void ProcessList::killAllQueries() { - auto lock = safeLock(); + std::vector cancelled_processes; + + SCOPE_EXIT({ + auto lock = safeLock(); + for (auto & cancelled_process : cancelled_processes) + cancelled_process->is_cancelling = false; + cancelled_cv.notify_all(); + }); + + { + auto lock = safeLock(); + cancelled_processes.reserve(processes.size()); + for (auto & process : processes) + { + cancelled_processes.push_back(&process); + process.is_cancelling = true; + } + } + + for (auto & cancelled_process : cancelled_processes) + cancelled_process->cancelQuery(true); - for (auto & process : processes) - process.cancelQuery(true); } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 203a1f114df..e7ad4e70712 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -100,6 +100,11 @@ protected: QueryPriorities::Handle priority_handle = nullptr; + /// True if query cancellation is in progress right now + /// ProcessListEntry should not be destroyed if is_cancelling is true + /// Flag changes is synced with ProcessListBase::mutex and notified with ProcessList::cancelled_cv + bool is_cancelling { false }; + /// KILL was send to the query std::atomic is_killed { false }; /// All data to the client already had been sent. @@ -331,6 +336,9 @@ protected: /// List of queries Container processes; + /// Notify about cancelled queries (done with ProcessListBase::mutex acquired). + mutable std::condition_variable cancelled_cv; + size_t max_size = 0; /// 0 means no limit. Otherwise, when limit exceeded, an exception is thrown. /// Stores per-user info: queries, statistics and limits diff --git a/src/Interpreters/SortedBlocksWriter.cpp b/src/Interpreters/SortedBlocksWriter.cpp index 82d501451a6..8f598f3dd3f 100644 --- a/src/Interpreters/SortedBlocksWriter.cpp +++ b/src/Interpreters/SortedBlocksWriter.cpp @@ -28,6 +28,11 @@ namespace CurrentMetrics namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace { @@ -84,10 +89,13 @@ void SortedBlocksWriter::insert(Block && block) size_t bytes = 0; size_t flush_no = 0; + if (!block.rows()) + return; + { std::lock_guard lock{insert_mutex}; - /// insert bock into BlocksList undef lock + /// insert block into BlocksList under lock inserted_blocks.insert(std::move(block)); size_t total_row_count = inserted_blocks.row_count + row_count_in_flush; @@ -145,7 +153,7 @@ SortedBlocksWriter::TmpFilePtr SortedBlocksWriter::flush(const BlocksList & bloc pipes.emplace_back(std::make_shared(block.cloneEmpty(), Chunk(block.getColumns(), num_rows))); if (pipes.empty()) - return {}; + throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty block"); QueryPipelineBuilder pipeline; pipeline.init(Pipe::unitePipes(std::move(pipes))); diff --git a/src/Interpreters/StorageID.h b/src/Interpreters/StorageID.h index c60c3aec9c6..43710988243 100644 --- a/src/Interpreters/StorageID.h +++ b/src/Interpreters/StorageID.h @@ -69,6 +69,8 @@ struct StorageID return uuid != UUIDHelpers::Nil; } + bool hasDatabase() const { return !database_name.empty(); } + bool operator<(const StorageID & rhs) const; bool operator==(const StorageID & rhs) const; diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 3f7e141db3e..74f084df40b 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -418,7 +418,7 @@ void optimizeDuplicateDistinct(ASTSelectQuery & select) return; std::unordered_set distinct_names = getDistinctNames(*subselect); - std::unordered_set selected_names; + std::unordered_set selected_names; /// Check source column names from select list (ignore aliases and table names) for (const auto & id : select.select()->children) @@ -427,11 +427,11 @@ void optimizeDuplicateDistinct(ASTSelectQuery & select) if (!identifier) return; - String name = identifier->shortName(); + const String & name = identifier->shortName(); if (!distinct_names.contains(name)) return; /// Not a distinct column, keep DISTINCT for it. - selected_names.insert(name); + selected_names.emplace(name); } /// select columns list != distinct columns list diff --git a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp index 8e4b66ef893..18784609397 100644 --- a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp +++ b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp @@ -28,16 +28,18 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +namespace +{ + class UserDefinedFunction final : public IFunction { public: - explicit UserDefinedFunction( ExternalUserDefinedExecutableFunctionsLoader::UserDefinedExecutableFunctionPtr executable_function_, ContextPtr context_, Array parameters_) : executable_function(std::move(executable_function_)) - , context(context_) + , context(std::move(context_)) { const auto & configuration = executable_function->getConfiguration(); size_t command_parameters_size = configuration.parameters.size(); @@ -230,6 +232,8 @@ private: std::vector command_arguments_with_parameters; }; +} + UserDefinedExecutableFunctionFactory & UserDefinedExecutableFunctionFactory::instance() { static UserDefinedExecutableFunctionFactory result; diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index dd23ad69ae2..4e7562ef451 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -6,26 +6,24 @@ #include #include #include -#include -#include #include #include #include -#include #include #include #include #include #include #include +#include #include + #include #include #include - +#include #include -#include namespace DB @@ -223,7 +221,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID && (which_from_type.isNativeInt() || which_from_type.isNativeUInt() || which_from_type.isDate() || which_from_type.isDate32() || which_from_type.isDateTime() || which_from_type.isDateTime64())) { const auto scale = static_cast(type).getScale(); - const auto decimal_value = DecimalUtils::decimalFromComponents(src.reinterpret(), 0, scale); + const auto decimal_value = DecimalUtils::decimalFromComponents(applyVisitor(FieldVisitorConvertToNumber(), src), 0, scale); return Field(DecimalField(decimal_value, scale)); } } diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 7cc4efcb64d..25e1dce4f9f 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -562,12 +562,16 @@ bool maybeRemoveOnCluster(const ASTPtr & query_ptr, ContextPtr context) if (database_name != query_on_cluster->cluster) return false; - auto db = DatabaseCatalog::instance().tryGetDatabase(database_name); - if (!db || db->getEngineName() != "Replicated") - return false; + auto database = DatabaseCatalog::instance().tryGetDatabase(database_name); + if (database && database->shouldReplicateQuery(context, query_ptr)) + { + /// It's Replicated database and query is replicated on database level, + /// so ON CLUSTER clause is redundant. + query_on_cluster->cluster.clear(); + return true; + } - query_on_cluster->cluster.clear(); - return true; + return false; } } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index c501c1722ba..0bca9a6ce6a 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -111,8 +112,11 @@ static String prepareQueryForLogging(const String & query, ContextPtr context) { String res = query; - // wiping sensitive data before cropping query by log_queries_cut_to_length, - // otherwise something like credit card without last digit can go to log + // Wiping a password or its hash from CREATE/ALTER USER query because we don't want it to go to logs. + res = wipePasswordFromQuery(res); + + // Wiping sensitive data before cropping query by log_queries_cut_to_length, + // otherwise something like credit card without last digit can go to log. if (auto * masker = SensitiveDataMasker::getInstance()) { auto matches = masker->wipeSensitiveData(res); @@ -838,101 +842,117 @@ static std::tuple executeQueryImpl( { QueryStatus * process_list_elem = context->getProcessListElement(); - if (!process_list_elem) - return; - - /// Update performance counters before logging to query_log - CurrentThread::finalizePerformanceCounters(); - - QueryStatusInfo info = process_list_elem->getInfo(true, context->getSettingsRef().log_profile_events); - - double elapsed_seconds = info.elapsed_seconds; - - elem.type = QueryLogElementType::QUERY_FINISH; - - // construct event_time and event_time_microseconds using the same time point - // so that the two times will always be equal up to a precision of a second. - const auto finish_time = std::chrono::system_clock::now(); - elem.event_time = time_in_seconds(finish_time); - elem.event_time_microseconds = time_in_microseconds(finish_time); - status_info_to_query_log(elem, info, ast, context); - - if (pulling_pipeline) + if (process_list_elem) { - query_pipeline.tryGetResultRowsAndBytes(elem.result_rows, elem.result_bytes); - } - else /// will be used only for ordinary INSERT queries - { - auto progress_out = process_list_elem->getProgressOut(); - elem.result_rows = progress_out.written_rows; - elem.result_bytes = progress_out.written_bytes; - } + /// Update performance counters before logging to query_log + CurrentThread::finalizePerformanceCounters(); - auto progress_callback = context->getProgressCallback(); - if (progress_callback) - { - Progress p(WriteProgress{info.written_rows, info.written_bytes}); - p.incrementPiecewiseAtomically(Progress{ResultProgress{elem.result_rows, elem.result_bytes}}); - progress_callback(p); - } + QueryStatusInfo info = process_list_elem->getInfo(true, context->getSettingsRef().log_profile_events); - if (elem.read_rows != 0) - { - LOG_INFO(&Poco::Logger::get("executeQuery"), "Read {} rows, {} in {} sec., {} rows/sec., {}/sec.", - elem.read_rows, ReadableSize(elem.read_bytes), elapsed_seconds, - static_cast(elem.read_rows / elapsed_seconds), - ReadableSize(elem.read_bytes / elapsed_seconds)); - } + double elapsed_seconds = info.elapsed_seconds; - if (log_queries && elem.type >= log_queries_min_type && static_cast(elem.query_duration_ms) >= log_queries_min_query_duration_ms) - { - if (auto query_log = context->getQueryLog()) - query_log->add(elem); - } - if (log_processors_profiles) - { - if (auto processors_profile_log = context->getProcessorsProfileLog()) + elem.type = QueryLogElementType::QUERY_FINISH; + + // construct event_time and event_time_microseconds using the same time point + // so that the two times will always be equal up to a precision of a second. + const auto finish_time = std::chrono::system_clock::now(); + elem.event_time = time_in_seconds(finish_time); + elem.event_time_microseconds = time_in_microseconds(finish_time); + status_info_to_query_log(elem, info, ast, context); + + if (pulling_pipeline) { - ProcessorProfileLogElement processor_elem; - processor_elem.event_time = time_in_seconds(finish_time); - processor_elem.event_time_microseconds = time_in_microseconds(finish_time); - processor_elem.query_id = elem.client_info.current_query_id; + query_pipeline.tryGetResultRowsAndBytes(elem.result_rows, elem.result_bytes); + } + else /// will be used only for ordinary INSERT queries + { + auto progress_out = process_list_elem->getProgressOut(); + elem.result_rows = progress_out.written_rows; + elem.result_bytes = progress_out.written_bytes; + } - auto get_proc_id = [](const IProcessor & proc) -> UInt64 - { - return reinterpret_cast(&proc); - }; + auto progress_callback = context->getProgressCallback(); + if (progress_callback) + { + Progress p(WriteProgress{info.written_rows, info.written_bytes}); + p.incrementPiecewiseAtomically(Progress{ResultProgress{elem.result_rows, elem.result_bytes}}); + progress_callback(p); + } - for (const auto & processor : query_pipeline.getProcessors()) + if (elem.read_rows != 0) + { + LOG_INFO(&Poco::Logger::get("executeQuery"), "Read {} rows, {} in {} sec., {} rows/sec., {}/sec.", + elem.read_rows, ReadableSize(elem.read_bytes), elapsed_seconds, + static_cast(elem.read_rows / elapsed_seconds), + ReadableSize(elem.read_bytes / elapsed_seconds)); + } + + if (log_queries && elem.type >= log_queries_min_type && static_cast(elem.query_duration_ms) >= log_queries_min_query_duration_ms) + { + if (auto query_log = context->getQueryLog()) + query_log->add(elem); + } + if (log_processors_profiles) + { + if (auto processors_profile_log = context->getProcessorsProfileLog()) { - std::vector parents; - for (const auto & port : processor->getOutputs()) + ProcessorProfileLogElement processor_elem; + processor_elem.event_time = time_in_seconds(finish_time); + processor_elem.event_time_microseconds = time_in_microseconds(finish_time); + processor_elem.query_id = elem.client_info.current_query_id; + + auto get_proc_id = [](const IProcessor & proc) -> UInt64 { - if (!port.isConnected()) - continue; - const IProcessor & next = port.getInputPort().getProcessor(); - parents.push_back(get_proc_id(next)); + return reinterpret_cast(&proc); + }; + + for (const auto & processor : query_pipeline.getProcessors()) + { + std::vector parents; + for (const auto & port : processor->getOutputs()) + { + if (!port.isConnected()) + continue; + const IProcessor & next = port.getInputPort().getProcessor(); + parents.push_back(get_proc_id(next)); + } + + processor_elem.id = get_proc_id(*processor); + processor_elem.parent_ids = std::move(parents); + + processor_elem.plan_step = reinterpret_cast(processor->getQueryPlanStep()); + processor_elem.plan_group = processor->getQueryPlanStepGroup(); + + processor_elem.processor_name = processor->getName(); + + processor_elem.elapsed_us = processor->getElapsedUs(); + processor_elem.input_wait_elapsed_us = processor->getInputWaitElapsedUs(); + processor_elem.output_wait_elapsed_us = processor->getOutputWaitElapsedUs(); + + auto stats = processor->getProcessorDataStats(); + processor_elem.input_rows = stats.input_rows; + processor_elem.input_bytes = stats.input_bytes; + processor_elem.output_rows = stats.output_rows; + processor_elem.output_bytes = stats.output_bytes; + + processors_profile_log->add(processor_elem); } + } + } - processor_elem.id = get_proc_id(*processor); - processor_elem.parent_ids = std::move(parents); - - processor_elem.plan_step = reinterpret_cast(processor->getQueryPlanStep()); - processor_elem.plan_group = processor->getQueryPlanStepGroup(); - - processor_elem.processor_name = processor->getName(); - - processor_elem.elapsed_us = processor->getElapsedUs(); - processor_elem.input_wait_elapsed_us = processor->getInputWaitElapsedUs(); - processor_elem.output_wait_elapsed_us = processor->getOutputWaitElapsedUs(); - - auto stats = processor->getProcessorDataStats(); - processor_elem.input_rows = stats.input_rows; - processor_elem.input_bytes = stats.input_bytes; - processor_elem.output_rows = stats.output_rows; - processor_elem.output_bytes = stats.output_bytes; - - processors_profile_log->add(processor_elem); + if (implicit_txn_control) + { + try + { + implicit_txn_control->executeCommit(context->getSessionContext()); + implicit_txn_control.reset(); + } + catch (const Exception &) + { + /// An exception might happen when trying to commit the transaction. For example we might get an immediate exception + /// because ZK is down and wait_changes_become_visible_after_commit_mode == WAIT_UNKNOWN + implicit_txn_control.reset(); + throw; } } } @@ -945,27 +965,11 @@ static std::tuple executeQueryImpl( query_span->addAttributeIfNotEmpty("clickhouse.tracestate", OpenTelemetry::CurrentContext().tracestate); query_span->addAttributeIfNotZero("clickhouse.read_rows", elem.read_rows); query_span->addAttributeIfNotZero("clickhouse.read_bytes", elem.read_bytes); - query_span->addAttributeIfNotZero("clickhouse.written_rows", info.written_rows); + query_span->addAttributeIfNotZero("clickhouse.written_rows", elem.written_rows); query_span->addAttributeIfNotZero("clickhouse.written_bytes", elem.written_bytes); query_span->addAttributeIfNotZero("clickhouse.memory_usage", elem.memory_usage); query_span->finish(); } - - if (implicit_txn_control) - { - try - { - implicit_txn_control->executeCommit(context->getSessionContext()); - implicit_txn_control.reset(); - } - catch (const Exception &) - { - /// An exception might happen when trying to commit the transaction. For example we might get an immediate exception - /// because ZK is down and wait_changes_become_visible_after_commit_mode == WAIT_UNKNOWN - implicit_txn_control.reset(); - throw; - } - } }; auto exception_callback = [elem, diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp index 6f4d7d5c525..e16647091ba 100644 --- a/src/Interpreters/getHeaderForProcessingStage.cpp +++ b/src/Interpreters/getHeaderForProcessingStage.cpp @@ -57,7 +57,7 @@ bool removeJoin(ASTSelectQuery & select, TreeRewriterResult & rewriter_result, C const size_t left_table_pos = 0; /// Test each argument of `and` function and select ones related to only left table std::shared_ptr new_conj = makeASTFunction("and"); - for (auto && node : collectConjunctions(where)) + for (auto && node : splitConjunctionsAst(where)) { if (membership_collector.getIdentsMembership(node) == left_table_pos) new_conj->arguments->children.push_back(std::move(node)); diff --git a/src/Interpreters/tests/gtest_lru_file_cache.cpp b/src/Interpreters/tests/gtest_lru_file_cache.cpp index 6460eeef8c5..22150b9f656 100644 --- a/src/Interpreters/tests/gtest_lru_file_cache.cpp +++ b/src/Interpreters/tests/gtest_lru_file_cache.cpp @@ -3,9 +3,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -64,7 +64,7 @@ void download(DB::FileSegmentPtr file_segment) fs::create_directories(subdir); std::string data(size, '0'); - file_segment->write(data.data(), size, file_segment->getDownloadOffset()); + file_segment->write(data.data(), size, file_segment->getCurrentWriteOffset()); } void prepareAndDownload(DB::FileSegmentPtr file_segment) @@ -89,6 +89,7 @@ TEST(FileCache, get) { if (fs::exists(cache_base_path)) fs::remove_all(cache_base_path); + fs::create_directories(cache_base_path); DB::ThreadStatus thread_status; @@ -109,7 +110,7 @@ TEST(FileCache, get) auto key = cache.hash("key1"); { - auto holder = cache.getOrSet(key, 0, 10, false); /// Add range [0, 9] + auto holder = cache.getOrSet(key, 0, 10, {}); /// Add range [0, 9] auto segments = fromHolder(holder); /// Range was not present in cache. It should be added in cache as one while file segment. ASSERT_EQ(segments.size(), 1); @@ -138,7 +139,7 @@ TEST(FileCache, get) { /// Want range [5, 14], but [0, 9] already in cache, so only [10, 14] will be put in cache. - auto holder = cache.getOrSet(key, 5, 10, false); + auto holder = cache.getOrSet(key, 5, 10, {}); auto segments = fromHolder(holder); ASSERT_EQ(segments.size(), 2); @@ -158,14 +159,14 @@ TEST(FileCache, get) ASSERT_EQ(cache.getUsedCacheSize(), 15); { - auto holder = cache.getOrSet(key, 9, 1, false); /// Get [9, 9] + auto holder = cache.getOrSet(key, 9, 1, {}); /// Get [9, 9] auto segments = fromHolder(holder); ASSERT_EQ(segments.size(), 1); assertRange(7, segments[0], DB::FileSegment::Range(0, 9), DB::FileSegment::State::DOWNLOADED); } { - auto holder = cache.getOrSet(key, 9, 2, false); /// Get [9, 10] + auto holder = cache.getOrSet(key, 9, 2, {}); /// Get [9, 10] auto segments = fromHolder(holder); ASSERT_EQ(segments.size(), 2); assertRange(8, segments[0], DB::FileSegment::Range(0, 9), DB::FileSegment::State::DOWNLOADED); @@ -173,15 +174,15 @@ TEST(FileCache, get) } { - auto holder = cache.getOrSet(key, 10, 1, false); /// Get [10, 10] + auto holder = cache.getOrSet(key, 10, 1, {}); /// Get [10, 10] auto segments = fromHolder(holder); ASSERT_EQ(segments.size(), 1); assertRange(10, segments[0], DB::FileSegment::Range(10, 14), DB::FileSegment::State::DOWNLOADED); } - complete(cache.getOrSet(key, 17, 4, false)); /// Get [17, 20] - complete(cache.getOrSet(key, 24, 3, false)); /// Get [24, 26] - /// complete(cache.getOrSet(key, 27, 1, false)); /// Get [27, 27] + complete(cache.getOrSet(key, 17, 4, {})); /// Get [17, 20] + complete(cache.getOrSet(key, 24, 3, {})); /// Get [24, 26] + /// completeWithState(cache.getOrSet(key, 27, 1, false)); /// Get [27, 27] /// Current cache: [__________][_____] [____] [___][] /// ^ ^^ ^ ^ ^ ^ ^^^ @@ -191,7 +192,7 @@ TEST(FileCache, get) ASSERT_EQ(cache.getUsedCacheSize(), 22); { - auto holder = cache.getOrSet(key, 0, 26, false); /// Get [0, 25] + auto holder = cache.getOrSet(key, 0, 26, {}); /// Get [0, 25] auto segments = fromHolder(holder); ASSERT_EQ(segments.size(), 6); @@ -225,14 +226,14 @@ TEST(FileCache, get) /// as max elements size is reached, next attempt to put something in cache should fail. /// This will also check that [27, 27] was indeed evicted. - auto holder1 = cache.getOrSet(key, 27, 1, false); + auto holder1 = cache.getOrSet(key, 27, 1, {}); auto segments_1 = fromHolder(holder1); /// Get [27, 27] ASSERT_EQ(segments_1.size(), 1); assertRange(17, segments_1[0], DB::FileSegment::Range(27, 27), DB::FileSegment::State::EMPTY); } { - auto holder = cache.getOrSet(key, 12, 10, false); /// Get [12, 21] + auto holder = cache.getOrSet(key, 12, 10, {}); /// Get [12, 21] auto segments = fromHolder(holder); ASSERT_EQ(segments.size(), 4); @@ -256,7 +257,7 @@ TEST(FileCache, get) ASSERT_EQ(cache.getFileSegmentsNum(), 5); { - auto holder = cache.getOrSet(key, 23, 5, false); /// Get [23, 28] + auto holder = cache.getOrSet(key, 23, 5, {}); /// Get [23, 28] auto segments = fromHolder(holder); ASSERT_EQ(segments.size(), 3); @@ -277,12 +278,12 @@ TEST(FileCache, get) /// 17 21 2324 26 28 { - auto holder5 = cache.getOrSet(key, 2, 3,false); /// Get [2, 4] + auto holder5 = cache.getOrSet(key, 2, 3, {}); /// Get [2, 4] auto s5 = fromHolder(holder5); ASSERT_EQ(s5.size(), 1); assertRange(25, s5[0], DB::FileSegment::Range(2, 4), DB::FileSegment::State::EMPTY); - auto holder1 = cache.getOrSet(key, 30, 2, false); /// Get [30, 31] + auto holder1 = cache.getOrSet(key, 30, 2, {}); /// Get [30, 31] auto s1 = fromHolder(holder1); ASSERT_EQ(s1.size(), 1); assertRange(26, s1[0], DB::FileSegment::Range(30, 31), DB::FileSegment::State::EMPTY); @@ -298,20 +299,20 @@ TEST(FileCache, get) /// ^ ^ ^ ^ ^ ^ ^ ^ /// 2 4 23 24 26 27 30 31 - auto holder2 = cache.getOrSet(key, 23, 1, false); /// Get [23, 23] + auto holder2 = cache.getOrSet(key, 23, 1, {}); /// Get [23, 23] auto s2 = fromHolder(holder2); ASSERT_EQ(s2.size(), 1); - auto holder3 = cache.getOrSet(key, 24, 3, false); /// Get [24, 26] + auto holder3 = cache.getOrSet(key, 24, 3, {}); /// Get [24, 26] auto s3 = fromHolder(holder3); ASSERT_EQ(s3.size(), 1); - auto holder4 = cache.getOrSet(key, 27, 1, false); /// Get [27, 27] + auto holder4 = cache.getOrSet(key, 27, 1, {}); /// Get [27, 27] auto s4 = fromHolder(holder4); ASSERT_EQ(s4.size(), 1); /// All cache is now unreleasable because pointers are still hold - auto holder6 = cache.getOrSet(key, 0, 40, false); + auto holder6 = cache.getOrSet(key, 0, 40, {}); auto f = fromHolder(holder6); ASSERT_EQ(f.size(), 9); @@ -332,7 +333,7 @@ TEST(FileCache, get) } { - auto holder = cache.getOrSet(key, 2, 3, false); /// Get [2, 4] + auto holder = cache.getOrSet(key, 2, 3, {}); /// Get [2, 4] auto segments = fromHolder(holder); ASSERT_EQ(segments.size(), 1); assertRange(31, segments[0], DB::FileSegment::Range(2, 4), DB::FileSegment::State::DOWNLOADED); @@ -343,7 +344,7 @@ TEST(FileCache, get) /// 2 4 23 24 26 27 30 31 { - auto holder = cache.getOrSet(key, 25, 5, false); /// Get [25, 29] + auto holder = cache.getOrSet(key, 25, 5, {}); /// Get [25, 29] auto segments = fromHolder(holder); ASSERT_EQ(segments.size(), 3); @@ -367,7 +368,7 @@ TEST(FileCache, get) DB::CurrentThread::QueryScope query_scope_holder_1(query_context_1); thread_status_1.attachQueryContext(query_context_1); - auto holder_2 = cache.getOrSet(key, 25, 5, false); /// Get [25, 29] once again. + auto holder_2 = cache.getOrSet(key, 25, 5, {}); /// Get [25, 29] once again. auto segments_2 = fromHolder(holder_2); ASSERT_EQ(segments.size(), 3); @@ -406,11 +407,11 @@ TEST(FileCache, get) { /// Now let's check the similar case but getting ERROR state after segment->wait(), when - /// state is changed not manually via segment->complete(state) but from destructor of holder + /// state is changed not manually via segment->completeWithState(state) but from destructor of holder /// and notify_all() is also called from destructor of holder. std::optional holder; - holder.emplace(cache.getOrSet(key, 3, 23, false)); /// Get [3, 25] + holder.emplace(cache.getOrSet(key, 3, 23, {})); /// Get [3, 25] auto segments = fromHolder(*holder); ASSERT_EQ(segments.size(), 3); @@ -436,7 +437,7 @@ TEST(FileCache, get) DB::CurrentThread::QueryScope query_scope_holder_1(query_context_1); thread_status_1.attachQueryContext(query_context_1); - auto holder_2 = cache.getOrSet(key, 3, 23, false); /// Get [3, 25] once again + auto holder_2 = cache.getOrSet(key, 3, 23, {}); /// Get [3, 25] once again auto segments_2 = fromHolder(*holder); ASSERT_EQ(segments_2.size(), 3); @@ -485,7 +486,7 @@ TEST(FileCache, get) cache2.initialize(); auto key = cache2.hash("key1"); - auto holder1 = cache2.getOrSet(key, 2, 28, false); /// Get [2, 29] + auto holder1 = cache2.getOrSet(key, 2, 28, {}); /// Get [2, 29] auto segments1 = fromHolder(holder1); ASSERT_EQ(segments1.size(), 5); @@ -506,7 +507,7 @@ TEST(FileCache, get) cache2.initialize(); auto key = cache2.hash("key1"); - auto holder1 = cache2.getOrSet(key, 0, 25, false); /// Get [0, 24] + auto holder1 = cache2.getOrSet(key, 0, 25, {}); /// Get [0, 24] auto segments1 = fromHolder(holder1); ASSERT_EQ(segments1.size(), 3); diff --git a/src/Loggers/CMakeLists.txt b/src/Loggers/CMakeLists.txt index 02a886765a1..1f32d747b01 100644 --- a/src/Loggers/CMakeLists.txt +++ b/src/Loggers/CMakeLists.txt @@ -2,9 +2,9 @@ add_headers_and_sources(loggers .) # Standard version depends on DBMS and works with text log add_library(loggers ${loggers_sources} ${loggers_headers}) -target_compile_definitions(loggers PUBLIC WITH_TEXT_LOG=1) target_link_libraries(loggers PRIVATE dbms clickhouse_common_io) # Lightweight version doesn't work with textlog and also doesn't depend on DBMS add_library(loggers_no_text_log ${loggers_sources} ${loggers_headers}) +target_compile_definitions(loggers_no_text_log PUBLIC WITHOUT_TEXT_LOG=1) target_link_libraries(loggers_no_text_log PRIVATE clickhouse_common_io) diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp index 97e26f2c038..b3bd8d588cd 100644 --- a/src/Loggers/Loggers.cpp +++ b/src/Loggers/Loggers.cpp @@ -10,7 +10,7 @@ #include #include -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG #include #endif @@ -34,7 +34,7 @@ static std::string createDirectory(const std::string & file) return path; } -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG void Loggers::setTextLog(std::shared_ptr log, int max_priority) { text_log = log; @@ -44,7 +44,7 @@ void Loggers::setTextLog(std::shared_ptr log, int max_priority) void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger /*_root*/, const std::string & cmd_name) { -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG if (split) if (auto log = text_log.lock()) split->addTextLog(log, text_log_max_priority); diff --git a/src/Loggers/Loggers.h b/src/Loggers/Loggers.h index 22b2b5e2c69..31a215aa9ce 100644 --- a/src/Loggers/Loggers.h +++ b/src/Loggers/Loggers.h @@ -7,7 +7,7 @@ #include #include "OwnSplitChannel.h" -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG namespace DB { class TextLog; @@ -29,7 +29,7 @@ public: /// Close log files. On next log write files will be reopened. void closeLogs(Poco::Logger & logger); -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG void setTextLog(std::shared_ptr log, int max_priority); #endif @@ -41,7 +41,7 @@ private: /// Previous value of logger element in config. It is used to reinitialize loggers whenever the value changed. std::string config_logger; -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG std::weak_ptr text_log; int text_log_max_priority = -1; #endif diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp index b1502cc4558..35a6d4ad86a 100644 --- a/src/Loggers/OwnSplitChannel.cpp +++ b/src/Loggers/OwnSplitChannel.cpp @@ -21,7 +21,7 @@ namespace DB void OwnSplitChannel::log(const Poco::Message & msg) { -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG auto logs_queue = CurrentThread::getInternalTextLogsQueue(); if (channels.empty() && (logs_queue == nullptr || !logs_queue->isNeeded(msg.getPriority(), msg.getSource()))) @@ -89,7 +89,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg) channel.first->log(msg); // ordinary child } -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG auto logs_queue = CurrentThread::getInternalTextLogsQueue(); /// Log to "TCP queue" if message is not too noisy @@ -150,7 +150,7 @@ void OwnSplitChannel::addChannel(Poco::AutoPtr channel, const std channels.emplace(name, ExtendedChannelPtrPair(std::move(channel), dynamic_cast(channel.get()))); } -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG void OwnSplitChannel::addTextLog(std::shared_ptr log, int max_priority) { std::lock_guard lock(text_log_mutex); diff --git a/src/Loggers/OwnSplitChannel.h b/src/Loggers/OwnSplitChannel.h index 16231c105b7..80305c1ccee 100644 --- a/src/Loggers/OwnSplitChannel.h +++ b/src/Loggers/OwnSplitChannel.h @@ -7,7 +7,7 @@ #include #include "ExtendedLogChannel.h" -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG namespace DB { class TextLog; @@ -30,7 +30,7 @@ public: /// Adds a child channel void addChannel(Poco::AutoPtr channel, const std::string & name); -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG void addTextLog(std::shared_ptr log, int max_priority); #endif @@ -47,7 +47,7 @@ private: std::mutex text_log_mutex; -#ifdef WITH_TEXT_LOG +#ifndef WITHOUT_TEXT_LOG std::weak_ptr text_log; std::atomic text_log_max_priority = -1; #endif diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index a6ff52b74b7..ab5137d0960 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -201,6 +201,10 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, if (!filesystem_cache_path.empty()) settings.ostr << (settings.hilite ? hilite_none : "") << " " << filesystem_cache_path; } + else if (type == Type::UNFREEZE) + { + settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(backup_name); + } } diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index f2c3fa8ece5..33f2fcb708c 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -47,6 +47,7 @@ public: RELOAD_FUNCTIONS, RELOAD_EMBEDDED_DICTIONARIES, RELOAD_CONFIG, + RELOAD_USERS, RELOAD_SYMBOLS, RESTART_DISK, STOP_MERGES, diff --git a/src/Parsers/Access/ASTCreateRoleQuery.h b/src/Parsers/Access/ASTCreateRoleQuery.h index 0e0773c8972..906ea683e1a 100644 --- a/src/Parsers/Access/ASTCreateRoleQuery.h +++ b/src/Parsers/Access/ASTCreateRoleQuery.h @@ -10,11 +10,11 @@ class ASTSettingsProfileElements; /** CREATE ROLE [IF NOT EXISTS | OR REPLACE] name - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * * ALTER ROLE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] */ class ASTCreateRoleQuery : public IAST, public ASTQueryWithOnCluster { diff --git a/src/Parsers/Access/ASTCreateSettingsProfileQuery.h b/src/Parsers/Access/ASTCreateSettingsProfileQuery.h index 64546bc7230..441ec0f5233 100644 --- a/src/Parsers/Access/ASTCreateSettingsProfileQuery.h +++ b/src/Parsers/Access/ASTCreateSettingsProfileQuery.h @@ -11,12 +11,12 @@ class ASTRolesOrUsersSet; /** CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] * * ALTER SETTINGS PROFILE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] */ class ASTCreateSettingsProfileQuery : public IAST, public ASTQueryWithOnCluster diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index c61f2cdd9fd..b8eb70e2041 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -19,7 +19,7 @@ class ASTSettingsProfileElements; * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [DEFAULT DATABASE database | NONE] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] * * ALTER USER [IF EXISTS] name @@ -28,7 +28,7 @@ class ASTSettingsProfileElements; * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [DEFAULT DATABASE database | NONE] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] */ class ASTCreateUserQuery : public IAST, public ASTQueryWithOnCluster diff --git a/src/Parsers/Access/ASTSettingsProfileElement.cpp b/src/Parsers/Access/ASTSettingsProfileElement.cpp index 23dba8a926f..76973c428b2 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.cpp +++ b/src/Parsers/Access/ASTSettingsProfileElement.cpp @@ -52,10 +52,24 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form << applyVisitor(FieldVisitorToString{}, max_value); } - if (readonly) + if (writability) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (*readonly ? " READONLY" : " WRITABLE") - << (settings.hilite ? IAST::hilite_none : ""); + switch (*writability) + { + case SettingConstraintWritability::WRITABLE: + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WRITABLE" + << (settings.hilite ? IAST::hilite_none : ""); + break; + case SettingConstraintWritability::CONST: + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CONST" + << (settings.hilite ? IAST::hilite_none : ""); + break; + case SettingConstraintWritability::CHANGEABLE_IN_READONLY: + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " CHANGEABLE_IN_READONLY" + << (settings.hilite ? IAST::hilite_none : ""); + break; + case SettingConstraintWritability::MAX: break; + } } } diff --git a/src/Parsers/Access/ASTSettingsProfileElement.h b/src/Parsers/Access/ASTSettingsProfileElement.h index 6a54bca3213..275257e4f8e 100644 --- a/src/Parsers/Access/ASTSettingsProfileElement.h +++ b/src/Parsers/Access/ASTSettingsProfileElement.h @@ -2,12 +2,12 @@ #include #include - +#include namespace DB { /** Represents a settings profile's element like the following - * {variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE]} | PROFILE 'profile_name' + * {variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY]} | PROFILE 'profile_name' */ class ASTSettingsProfileElement : public IAST { @@ -17,7 +17,7 @@ public: Field value; Field min_value; Field max_value; - std::optional readonly; + std::optional writability; bool id_mode = false; /// If true then `parent_profile` keeps UUID, not a name. bool use_inherit_keyword = false; /// If true then this element is a part of ASTCreateSettingsProfileQuery. @@ -30,7 +30,7 @@ public: /** Represents settings profile's elements like the following - * {{variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE]} | PROFILE 'profile_name'} [,...] + * {{variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY]} | PROFILE 'profile_name'} [,...] */ class ASTSettingsProfileElements : public IAST { diff --git a/src/Parsers/Access/ParserCreateRoleQuery.h b/src/Parsers/Access/ParserCreateRoleQuery.h index 1fdee67eaab..883ea89854a 100644 --- a/src/Parsers/Access/ParserCreateRoleQuery.h +++ b/src/Parsers/Access/ParserCreateRoleQuery.h @@ -7,11 +7,11 @@ namespace DB { /** Parses queries like * CREATE ROLE [IF NOT EXISTS | OR REPLACE] name - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * * ALTER ROLE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] */ class ParserCreateRoleQuery : public IParserBase { diff --git a/src/Parsers/Access/ParserCreateSettingsProfileQuery.h b/src/Parsers/Access/ParserCreateSettingsProfileQuery.h index ab730fcd8eb..bee5bdcb2d1 100644 --- a/src/Parsers/Access/ParserCreateSettingsProfileQuery.h +++ b/src/Parsers/Access/ParserCreateSettingsProfileQuery.h @@ -7,11 +7,11 @@ namespace DB { /** Parses queries like * CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] * * ALTER SETTINGS PROFILE [IF EXISTS] name * [RENAME TO new_name] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] */ class ParserCreateSettingsProfileQuery : public IParserBase { diff --git a/src/Parsers/Access/ParserCreateUserQuery.h b/src/Parsers/Access/ParserCreateUserQuery.h index 215133a777c..0cc8c9b6649 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.h +++ b/src/Parsers/Access/ParserCreateUserQuery.h @@ -10,7 +10,7 @@ namespace DB * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] * * ALTER USER [IF EXISTS] name @@ -18,7 +18,7 @@ namespace DB * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] */ class ParserCreateUserQuery : public IParserBase diff --git a/src/Parsers/Access/ParserSettingsProfileElement.cpp b/src/Parsers/Access/ParserSettingsProfileElement.cpp index f4ca52dea2a..db23a806a12 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.cpp +++ b/src/Parsers/Access/ParserSettingsProfileElement.cpp @@ -95,18 +95,23 @@ namespace } - bool parseReadonlyOrWritableKeyword(IParserBase::Pos & pos, Expected & expected, std::optional & readonly) + bool parseConstraintWritabilityKeyword(IParserBase::Pos & pos, Expected & expected, std::optional & writability) { return IParserBase::wrapParseImpl(pos, [&] { - if (ParserKeyword{"READONLY"}.ignore(pos, expected)) + if (ParserKeyword{"READONLY"}.ignore(pos, expected) || ParserKeyword{"CONST"}.ignore(pos, expected)) { - readonly = true; + writability = SettingConstraintWritability::CONST; return true; } else if (ParserKeyword{"WRITABLE"}.ignore(pos, expected)) { - readonly = false; + writability = SettingConstraintWritability::WRITABLE; + return true; + } + else if (ParserKeyword{"CHANGEABLE_IN_READONLY"}.ignore(pos, expected)) + { + writability = SettingConstraintWritability::CHANGEABLE_IN_READONLY; return true; } else @@ -122,7 +127,7 @@ namespace Field & value, Field & min_value, Field & max_value, - std::optional & readonly) + std::optional & writability) { return IParserBase::wrapParseImpl(pos, [&] { @@ -134,11 +139,11 @@ namespace Field res_value; Field res_min_value; Field res_max_value; - std::optional res_readonly; + std::optional res_writability; bool has_value_or_constraint = false; while (parseValue(pos, expected, res_value) || parseMinMaxValue(pos, expected, res_min_value, res_max_value) - || parseReadonlyOrWritableKeyword(pos, expected, res_readonly)) + || parseConstraintWritabilityKeyword(pos, expected, res_writability)) { has_value_or_constraint = true; } @@ -147,7 +152,7 @@ namespace return false; if (boost::iequals(res_setting_name, "PROFILE") && res_value.isNull() && res_min_value.isNull() && res_max_value.isNull() - && res_readonly) + && res_writability == SettingConstraintWritability::CONST) { /// Ambiguity: "profile readonly" can be treated either as a profile named "readonly" or /// as a setting named 'profile' with the readonly constraint. @@ -159,7 +164,7 @@ namespace value = std::move(res_value); min_value = std::move(res_min_value); max_value = std::move(res_max_value); - readonly = res_readonly; + writability = res_writability; return true; }); } @@ -179,9 +184,9 @@ namespace Field value; Field min_value; Field max_value; - std::optional readonly; + std::optional writability; - bool ok = parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, readonly); + bool ok = parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, writability); if (!ok && (parseProfileKeyword(pos, expected, use_inherit_keyword) || previous_element_was_parent_profile)) ok = parseProfileNameOrID(pos, expected, id_mode, parent_profile); @@ -195,7 +200,7 @@ namespace result->value = std::move(value); result->min_value = std::move(min_value); result->max_value = std::move(max_value); - result->readonly = readonly; + result->writability = writability; result->id_mode = id_mode; result->use_inherit_keyword = use_inherit_keyword; return true; diff --git a/src/Parsers/Access/ParserSettingsProfileElement.h b/src/Parsers/Access/ParserSettingsProfileElement.h index 8843591a56c..082fc66625f 100644 --- a/src/Parsers/Access/ParserSettingsProfileElement.h +++ b/src/Parsers/Access/ParserSettingsProfileElement.h @@ -6,7 +6,7 @@ namespace DB { /** Parses a string like this: - * {variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE]} | PROFILE 'profile_name' + * {variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY]} | PROFILE 'profile_name' */ class ParserSettingsProfileElement : public IParserBase { diff --git a/src/Parsers/wipePasswordFromQuery.cpp b/src/Parsers/wipePasswordFromQuery.cpp new file mode 100644 index 00000000000..87b297d6fcc --- /dev/null +++ b/src/Parsers/wipePasswordFromQuery.cpp @@ -0,0 +1,32 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +String wipePasswordFromQuery(const String & query) +{ + String error_message; + const char * begin = query.data(); + const char * end = begin + query.size(); + + { + ParserCreateUserQuery parser; + const char * pos = begin; + if (auto ast = tryParseQuery(parser, pos, end, error_message, false, "", false, 0, 0)) + { + auto create_query = typeid_cast>(ast); + create_query->show_password = false; + return serializeAST(*create_query); + } + } + + return query; +} + +} diff --git a/src/Parsers/wipePasswordFromQuery.h b/src/Parsers/wipePasswordFromQuery.h new file mode 100644 index 00000000000..cdd8518de53 --- /dev/null +++ b/src/Parsers/wipePasswordFromQuery.h @@ -0,0 +1,14 @@ +#pragma once +#include + + +namespace DB +{ + +/// Removes a password or its hash from a query if it's specified there or replaces it with some placeholder. +/// This function is used to prepare a query for storing in logs (we don't want logs to contain sensitive information). +/// The function changes only following types of queries: +/// CREATE/ALTER USER. +String wipePasswordFromQuery(const String & query); + +} diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index e2bd208764e..091447e96ee 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -55,9 +56,13 @@ public: void addBuffer(std::unique_ptr buffer) { owned_buffers.emplace_back(std::move(buffer)); } + void setErrorsLogger(const InputFormatErrorsLoggerPtr & errors_logger_) { errors_logger = errors_logger_; } + protected: ColumnMappingPtr column_mapping{}; + InputFormatErrorsLoggerPtr errors_logger; + private: /// Number of currently parsed chunk (if parallel parsing is enabled) size_t current_unit_number = 0; diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index 3df22002b82..52395338279 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -52,6 +52,31 @@ IRowInputFormat::IRowInputFormat(Block header, ReadBuffer & in_, Params params_) { } +void IRowInputFormat::logError() +{ + String diagnostic; + String raw_data; + try + { + std::tie(diagnostic, raw_data) = getDiagnosticAndRawData(); + } + catch (const Exception & exception) + { + diagnostic = "Cannot get diagnostic: " + exception.message(); + raw_data = "Cannot get raw data: " + exception.message(); + } + catch (...) + { + /// Error while trying to obtain verbose diagnostic. Ok to ignore. + } + trimLeft(diagnostic, '\n'); + trimRight(diagnostic, '\n'); + + auto now_time = time(nullptr); + + errors_logger->logError(InputFormatErrorsLogger::ErrorEntry{now_time, total_rows, diagnostic, raw_data}); +} + Chunk IRowInputFormat::generate() { if (total_rows == 0) @@ -112,6 +137,9 @@ Chunk IRowInputFormat::generate() if (params.allow_errors_num == 0 && params.allow_errors_ratio == 0) throw; + if (errors_logger) + logError(); + ++num_errors; Float64 current_error_ratio = static_cast(num_errors) / total_rows; diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index 87caadd93da..a11462549ff 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -65,6 +65,10 @@ protected: /// and collect as much as possible diagnostic information about error. /// If not implemented, returns empty string. virtual std::string getDiagnosticInfo() { return {}; } + /// Get diagnostic info and raw data for a row + virtual std::pair getDiagnosticAndRawData() { return std::make_pair("", ""); } + + void logError(); const BlockMissingValues & getMissingValues() const override { return block_missing_values; } diff --git a/src/Processors/Formats/IRowOutputFormat.cpp b/src/Processors/Formats/IRowOutputFormat.cpp index f2f6b49ed3f..52ce9c1b227 100644 --- a/src/Processors/Formats/IRowOutputFormat.cpp +++ b/src/Processors/Formats/IRowOutputFormat.cpp @@ -40,6 +40,9 @@ void IRowOutputFormat::consume(DB::Chunk chunk) void IRowOutputFormat::consumeTotals(DB::Chunk chunk) { + if (!supportTotals()) + return; + auto num_rows = chunk.getNumRows(); if (num_rows != 1) throw Exception("Got " + toString(num_rows) + " in totals chunk, expected 1", ErrorCodes::LOGICAL_ERROR); @@ -53,6 +56,9 @@ void IRowOutputFormat::consumeTotals(DB::Chunk chunk) void IRowOutputFormat::consumeExtremes(DB::Chunk chunk) { + if (!supportExtremes()) + return; + auto num_rows = chunk.getNumRows(); const auto & columns = chunk.getColumns(); if (num_rows != 2) diff --git a/src/Processors/Formats/IRowOutputFormat.h b/src/Processors/Formats/IRowOutputFormat.h index 7a57753d765..da2eb192e90 100644 --- a/src/Processors/Formats/IRowOutputFormat.h +++ b/src/Processors/Formats/IRowOutputFormat.h @@ -32,6 +32,9 @@ protected: void consumeTotals(Chunk chunk) override; void consumeExtremes(Chunk chunk) override; + virtual bool supportTotals() const { return false; } + virtual bool supportExtremes() const { return false; } + /** Write a row. * Default implementation calls methods to write single values and delimiters * (except delimiter between rows (writeRowBetweenDelimiter())). diff --git a/src/Processors/Formats/ISchemaReader.cpp b/src/Processors/Formats/ISchemaReader.cpp index c7d8b87ab77..9365384f4b7 100644 --- a/src/Processors/Formats/ISchemaReader.cpp +++ b/src/Processors/Formats/ISchemaReader.cpp @@ -113,6 +113,11 @@ NamesAndTypesList IRowSchemaReader::readSchema() "Most likely setting input_format_max_rows_to_read_for_schema_inference is set to 0"); DataTypes data_types = readRowAndGetDataTypes(); + + /// Check that we read at list one column. + if (data_types.empty()) + throw Exception(ErrorCodes::EMPTY_DATA_PASSED, "Cannot read rows from the data"); + /// If column names weren't set, use default names 'c1', 'c2', ... if (column_names.empty()) { @@ -122,9 +127,11 @@ NamesAndTypesList IRowSchemaReader::readSchema() } /// If column names were set, check that the number of names match the number of types. else if (column_names.size() != data_types.size()) + { throw Exception( ErrorCodes::INCORRECT_DATA, "The number of column names {} differs with the number of types {}", column_names.size(), data_types.size()); + } for (size_t i = 0; i != column_names.size(); ++i) { @@ -155,10 +162,6 @@ NamesAndTypesList IRowSchemaReader::readSchema() } } - /// Check that we read at list one column. - if (data_types.empty()) - throw Exception(ErrorCodes::EMPTY_DATA_PASSED, "Cannot read rows from the data"); - NamesAndTypesList result; for (size_t i = 0; i != data_types.size(); ++i) { diff --git a/src/Processors/Formats/Impl/CSVRowOutputFormat.h b/src/Processors/Formats/Impl/CSVRowOutputFormat.h index a36c5ff47fb..0efc00378e5 100644 --- a/src/Processors/Formats/Impl/CSVRowOutputFormat.h +++ b/src/Processors/Formats/Impl/CSVRowOutputFormat.h @@ -34,6 +34,10 @@ private: void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; void writeFieldDelimiter() override; void writeRowEndDelimiter() override; + + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h index 63e9e9f1b76..2b4c596d4b5 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h @@ -36,9 +36,8 @@ private: void writeRowStartDelimiter() override; void writeRowEndDelimiter() override; + bool supportTotals() const override { return true; } void consumeTotals(Chunk) override; - /// No extremes. - void consumeExtremes(Chunk) override {} void writeLine(const std::vector & values); diff --git a/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h index d17a6acf019..38123833f10 100644 --- a/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h @@ -31,6 +31,9 @@ private: void writeRowStartDelimiter() override; void writeRowEndDelimiter() override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeAfterTotals() override; diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h index ac03c2991bf..37a7029d050 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h @@ -39,10 +39,6 @@ protected: void writePrefix() override; void writeSuffix() override; - /// No totals and extremes. - void consumeTotals(Chunk) override {} - void consumeExtremes(Chunk) override {} - size_t field_number = 0; private: diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.h b/src/Processors/Formats/Impl/JSONRowOutputFormat.h index 66b8d0f682e..07c526885f6 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.h @@ -56,6 +56,9 @@ protected: void writeMaxExtreme(const Columns & columns, size_t row_num) override; void writeTotals(const Columns & columns, size_t row_num) override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeAfterTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index b3d237fecfd..bfc4f726edb 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -2,6 +2,15 @@ #if USE_MSGPACK +/// FIXME: there is some issue with clang-15, that incorrectly detect a +/// "Attempt to free released memory" in msgpack::unpack(), because of delete +/// operator for zone (from msgpack/v1/detail/cpp11_zone.hpp), hence NOLINT +/// +/// NOTE: that I was not able to suppress it locally, only with +/// NOLINTBEGIN/NOLINTEND +// +// NOLINTBEGIN(clang-analyzer-cplusplus.NewDelete) + #include #include #include @@ -235,8 +244,10 @@ static void insertNull(IColumn & column, DataTypePtr type) assert_cast(column).insertDefault(); } -static void insertUUID(IColumn & column, DataTypePtr /*type*/, const char * value, size_t size) +static void insertUUID(IColumn & column, DataTypePtr type, const char * value, size_t size) { + if (!isUUID(type)) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert MessagePack UUID into column with type {}.", type->getName()); ReadBufferFromMemory buf(value, size); UUID uuid; readBinaryBigEndian(uuid.toUnderType().items[0], buf); @@ -551,6 +562,8 @@ void registerMsgPackSchemaReader(FormatFactory & factory) } +// NOLINTEND(clang-analyzer-cplusplus.NewDelete) + #else namespace DB diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index 1de2acbb3b9..6dd5d37e458 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -55,7 +55,7 @@ ORCBlockOutputFormat::ORCBlockOutputFormat(WriteBuffer & out_, const Block & hea data_types.push_back(recursiveRemoveLowCardinality(type)); } -ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & type) +std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & type) { switch (type->getTypeId()) { @@ -398,17 +398,21 @@ void ORCBlockOutputFormat::writeColumn( const auto & list_column = assert_cast(column); auto nested_type = assert_cast(*type).getNestedType(); const ColumnArray::Offsets & offsets = list_column.getOffsets(); - list_orc_column.resize(list_column.size()); + + size_t column_size = list_column.size(); + list_orc_column.resize(column_size); + /// The length of list i in ListVectorBatch is offsets[i+1] - offsets[i]. list_orc_column.offsets[0] = 0; - for (size_t i = 0; i != list_column.size(); ++i) + for (size_t i = 0; i != column_size; ++i) { list_orc_column.offsets[i + 1] = offsets[i]; list_orc_column.notNull[i] = 1; } + orc::ColumnVectorBatch & nested_orc_column = *list_orc_column.elements; writeColumn(nested_orc_column, list_column.getData(), nested_type, null_bytemap); - list_orc_column.numElements = list_column.size(); + list_orc_column.numElements = column_size; break; } case TypeIndex::Tuple: @@ -429,10 +433,12 @@ void ORCBlockOutputFormat::writeColumn( const auto & map_type = assert_cast(*type); const ColumnArray::Offsets & offsets = list_column.getOffsets(); + size_t column_size = list_column.size(); + map_orc_column.resize(list_column.size()); /// The length of list i in ListVectorBatch is offsets[i+1] - offsets[i]. map_orc_column.offsets[0] = 0; - for (size_t i = 0; i != list_column.size(); ++i) + for (size_t i = 0; i != column_size; ++i) { map_orc_column.offsets[i + 1] = offsets[i]; map_orc_column.notNull[i] = 1; @@ -447,7 +453,7 @@ void ORCBlockOutputFormat::writeColumn( auto value_type = map_type.getValueType(); writeColumn(values_orc_column, *nested_columns[1], value_type, null_bytemap); - map_orc_column.numElements = list_column.size(); + map_orc_column.numElements = column_size; break; } default: @@ -461,8 +467,9 @@ size_t ORCBlockOutputFormat::getColumnSize(const IColumn & column, DataTypePtr & { auto nested_type = assert_cast(*type).getNestedType(); const IColumn & nested_column = assert_cast(column).getData(); - return getColumnSize(nested_column, nested_type); + return std::max(column.size(), getColumnSize(nested_column, nested_type)); } + return column.size(); } @@ -471,9 +478,7 @@ size_t ORCBlockOutputFormat::getMaxColumnSize(Chunk & chunk) size_t columns_num = chunk.getNumColumns(); size_t max_column_size = 0; for (size_t i = 0; i != columns_num; ++i) - { max_column_size = std::max(max_column_size, getColumnSize(*chunk.getColumns()[i], data_types[i])); - } return max_column_size; } @@ -481,18 +486,23 @@ void ORCBlockOutputFormat::consume(Chunk chunk) { if (!writer) prepareWriter(); + size_t columns_num = chunk.getNumColumns(); size_t rows_num = chunk.getNumRows(); + /// getMaxColumnSize is needed to write arrays. - /// The size of the batch must be no less than total amount of array elements. - ORC_UNIQUE_PTR batch = writer->createRowBatch(getMaxColumnSize(chunk)); + /// The size of the batch must be no less than total amount of array elements + /// and no less than the number of rows (ORC writes a null bit for every row). + std::unique_ptr batch = writer->createRowBatch(getMaxColumnSize(chunk)); orc::StructVectorBatch & root = dynamic_cast(*batch); + auto columns = chunk.detachColumns(); for (auto & column : columns) column = recursiveRemoveLowCardinality(column); for (size_t i = 0; i != columns_num; ++i) writeColumn(*root.fields[i], *columns[i], data_types[i], nullptr); + root.numElements = rows_num; writer->add(*batch); } diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h index d4a19353915..6467f2148f5 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h @@ -8,11 +8,13 @@ #include #include + namespace DB { class WriteBuffer; + /// orc::Writer writes only in orc::OutputStream class ORCOutputStream : public orc::OutputStream { @@ -21,7 +23,7 @@ public: uint64_t getLength() const override; uint64_t getNaturalWriteSize() const override; - void write(const void* buf, size_t length) override; + void write(const void * buf, size_t length) override; void close() override {} const std::string& getName() const override { return name; } @@ -31,6 +33,7 @@ private: std::string name = "ORCOutputStream"; }; + class ORCBlockOutputFormat : public IOutputFormat { public: @@ -42,7 +45,7 @@ private: void consume(Chunk chunk) override; void finalizeImpl() override; - ORC_UNIQUE_PTR getORCType(const DataTypePtr & type); + std::unique_ptr getORCType(const DataTypePtr & type); /// ConvertFunc is needed for type UInt8, because firstly UInt8 (char8_t) must be /// converted to unsigned char (bugprone-signed-char-misuse in clang). @@ -75,8 +78,8 @@ private: const FormatSettings format_settings; ORCOutputStream output_stream; DataTypes data_types; - ORC_UNIQUE_PTR writer; - ORC_UNIQUE_PTR schema; + std::unique_ptr writer; + std::unique_ptr schema; orc::WriterOptions options; }; diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 318bcaed466..e0693b489bd 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -75,6 +75,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa InputFormatPtr input_format = internal_parser_creator(read_buffer); input_format->setCurrentUnitNumber(current_ticket_number); + input_format->setErrorsLogger(errors_logger); InternalParser parser(input_format); unit.chunk_ext.chunk.clear(); diff --git a/src/Processors/Formats/Impl/TSKVRowOutputFormat.h b/src/Processors/Formats/Impl/TSKVRowOutputFormat.h index e9f9071f906..9bc44bbb4d7 100644 --- a/src/Processors/Formats/Impl/TSKVRowOutputFormat.h +++ b/src/Processors/Formats/Impl/TSKVRowOutputFormat.h @@ -22,6 +22,10 @@ private: void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; void writeRowEndDelimiter() override; + /// Disable totals and extremes, because they are enabled in TSV. + bool supportTotals() const override { return false; } + bool supportExtremes() const override { return false; } + NamesAndTypes fields; size_t field_number = 0; }; diff --git a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h index 8781b7be0b1..533e30a3ee1 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h +++ b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h @@ -37,6 +37,10 @@ protected: void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; void writeFieldDelimiter() override final; void writeRowEndDelimiter() override; + + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override final; void writeBeforeExtremes() override final; diff --git a/src/Processors/Formats/Impl/VerticalRowOutputFormat.h b/src/Processors/Formats/Impl/VerticalRowOutputFormat.h index 796c60d1cb1..c3dbafd8b9b 100644 --- a/src/Processors/Formats/Impl/VerticalRowOutputFormat.h +++ b/src/Processors/Formats/Impl/VerticalRowOutputFormat.h @@ -32,6 +32,9 @@ private: void writeMaxExtreme(const Columns & columns, size_t row_num) override; void writeTotals(const Columns & columns, size_t row_num) override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/Impl/XMLRowOutputFormat.h b/src/Processors/Formats/Impl/XMLRowOutputFormat.h index abec25efbb9..4d2f34b0bfe 100644 --- a/src/Processors/Formats/Impl/XMLRowOutputFormat.h +++ b/src/Processors/Formats/Impl/XMLRowOutputFormat.h @@ -32,6 +32,9 @@ private: void writeMaxExtreme(const Columns & columns, size_t row_num) override; void writeTotals(const Columns & columns, size_t row_num) override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeAfterTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/InputFormatErrorsLogger.cpp b/src/Processors/Formats/InputFormatErrorsLogger.cpp new file mode 100644 index 00000000000..e6f8cdd43ee --- /dev/null +++ b/src/Processors/Formats/InputFormatErrorsLogger.cpp @@ -0,0 +1,83 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace +{ + const String DEFAULT_OUTPUT_FORMAT = "CSV"; +} + +InputFormatErrorsLogger::InputFormatErrorsLogger(const ContextPtr & context) +{ + String output_format = context->getSettingsRef().errors_output_format; + if (!FormatFactory::instance().isOutputFormat(output_format)) + output_format = DEFAULT_OUTPUT_FORMAT; + if (context->hasInsertionTable()) + table = context->getInsertionTable().getTableName(); + if (context->getInsertionTable().hasDatabase()) + database = context->getInsertionTable().getDatabaseName(); + + String path_in_setting = context->getSettingsRef().input_format_record_errors_file_path; + errors_file_path = context->getApplicationType() == Context::ApplicationType::SERVER ? context->getUserFilesPath() + path_in_setting + : path_in_setting; + while (fs::exists(errors_file_path)) + { + errors_file_path += "_new"; + } + write_buf = std::make_shared(errors_file_path); + + header = Block{ + {std::make_shared(), "time"}, + {std::make_shared(std::make_shared()), "database"}, + {std::make_shared(std::make_shared()), "table"}, + {std::make_shared(), "offset"}, + {std::make_shared(), "reason"}, + {std::make_shared(), "raw_data"}}; + + writer = context->getOutputFormat(output_format, *write_buf, header); +} + +InputFormatErrorsLogger::~InputFormatErrorsLogger() +{ + writer->finalize(); + writer->flush(); + write_buf->finalize(); +} + +void InputFormatErrorsLogger::logErrorImpl(ErrorEntry entry) +{ + auto error = header.cloneEmpty(); + auto columns = error.mutateColumns(); + columns[0]->insert(entry.time); + database.empty() ? columns[1]->insertDefault() : columns[1]->insert(database); + table.empty() ? columns[2]->insertDefault() : columns[2]->insert(table); + columns[3]->insert(entry.offset); + columns[4]->insert(entry.reason); + columns[5]->insert(entry.raw_data); + error.setColumns(std::move(columns)); + + writer->write(error); +} + +void InputFormatErrorsLogger::logError(ErrorEntry entry) +{ + logErrorImpl(entry); +} + +ParallelInputFormatErrorsLogger::~ParallelInputFormatErrorsLogger() = default; + +void ParallelInputFormatErrorsLogger::logError(ErrorEntry entry) +{ + std::lock_guard lock(write_mutex); + logErrorImpl(entry); +} + +} diff --git a/src/Processors/Formats/InputFormatErrorsLogger.h b/src/Processors/Formats/InputFormatErrorsLogger.h new file mode 100644 index 00000000000..4b3766f4d37 --- /dev/null +++ b/src/Processors/Formats/InputFormatErrorsLogger.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class InputFormatErrorsLogger +{ +public: + struct ErrorEntry + { + time_t time; + size_t offset; + String reason; + String raw_data; + }; + + InputFormatErrorsLogger(const ContextPtr & context); + + virtual ~InputFormatErrorsLogger(); + + virtual void logError(ErrorEntry entry); + void logErrorImpl(ErrorEntry entry); + +private: + Block header; + + String errors_file_path; + std::shared_ptr write_buf; + OutputFormatPtr writer; + + String database; + String table; +}; + +using InputFormatErrorsLoggerPtr = std::shared_ptr; + +class ParallelInputFormatErrorsLogger : public InputFormatErrorsLogger +{ +public: + ParallelInputFormatErrorsLogger(const ContextPtr & context) : InputFormatErrorsLogger(context) { } + + ~ParallelInputFormatErrorsLogger() override; + + void logError(ErrorEntry entry) override; + +private: + std::mutex write_mutex; +}; + +} diff --git a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp index f4568830720..35a86bc476d 100644 --- a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp +++ b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp @@ -35,12 +35,15 @@ void RowInputFormatWithDiagnosticInfo::updateDiagnosticInfo() offset_of_current_row = in->offset(); } -String RowInputFormatWithDiagnosticInfo::getDiagnosticInfo() +std::pair RowInputFormatWithDiagnosticInfo::getDiagnosticAndRawDataImpl(bool is_errors_record) { - if (in->eof()) - return "Buffer has gone, cannot extract information about what has been parsed."; + WriteBufferFromOwnString out_diag; + WriteBufferFromOwnString out_data; - WriteBufferFromOwnString out; + if (in->eof()) + return std::make_pair( + "Buffer has gone, cannot extract information about what has been parsed.", + "Buffer has gone, cannot extract information about what has been parsed."); const auto & header = getPort().getHeader(); MutableColumns columns = header.cloneEmptyColumns(); @@ -49,8 +52,9 @@ String RowInputFormatWithDiagnosticInfo::getDiagnosticInfo() size_t bytes_read_at_start_of_buffer = in->count() - in->offset(); if (bytes_read_at_start_of_buffer != bytes_read_at_start_of_buffer_on_prev_row) { - out << "Could not print diagnostic info because two last rows aren't in buffer (rare case)\n"; - return out.str(); + out_diag << "Could not print diagnostic info because two last rows aren't in buffer (rare case)\n"; + out_data << "Could not collect raw data because two last rows aren't in buffer (rare case)\n"; + return std::make_pair(out_diag.str(), out_data.str()); } max_length_of_column_name = 0; @@ -65,30 +69,49 @@ String RowInputFormatWithDiagnosticInfo::getDiagnosticInfo() /// Roll back the cursor to the beginning of the previous or current row and parse all over again. But now we derive detailed information. - if (offset_of_prev_row <= in->buffer().size()) + if (!is_errors_record && offset_of_prev_row <= in->buffer().size()) { in->position() = in->buffer().begin() + offset_of_prev_row; - out << "\nRow " << (row_num - 1) << ":\n"; - if (!parseRowAndPrintDiagnosticInfo(columns, out)) - return out.str(); + out_diag << "\nRow " << (row_num - 1) << ":\n"; + if (!parseRowAndPrintDiagnosticInfo(columns, out_diag)) + return std::make_pair(out_diag.str(), out_data.str()); } else { if (in->buffer().size() < offset_of_current_row) { - out << "Could not print diagnostic info because parsing of data hasn't started.\n"; - return out.str(); + out_diag << "Could not print diagnostic info because parsing of data hasn't started.\n"; + out_data << "Could not collect raw data because parsing of data hasn't started.\n"; + return std::make_pair(out_diag.str(), out_data.str()); } in->position() = in->buffer().begin() + offset_of_current_row; } - out << "\nRow " << row_num << ":\n"; - parseRowAndPrintDiagnosticInfo(columns, out); - out << "\n"; + char * data = in->position(); + while (data < in->buffer().end() && *data != '\n' && *data != '\r' && *data != '\0') + { + out_data << *data; + ++data; + } - return out.str(); + out_diag << "\nRow " << row_num << ":\n"; + parseRowAndPrintDiagnosticInfo(columns, out_diag); + out_diag << "\n"; + + return std::make_pair(out_diag.str(), out_data.str()); +} + +String RowInputFormatWithDiagnosticInfo::getDiagnosticInfo() +{ + auto diagnostic_and_raw_data = getDiagnosticAndRawDataImpl(false); + return std::get<0>(diagnostic_and_raw_data); +} + +std::pair RowInputFormatWithDiagnosticInfo::getDiagnosticAndRawData() +{ + return getDiagnosticAndRawDataImpl(true); } bool RowInputFormatWithDiagnosticInfo::deserializeFieldAndPrintDiagnosticInfo(const String & col_name, diff --git a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.h b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.h index 5bad24cd482..49793fcd208 100644 --- a/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.h +++ b/src/Processors/Formats/RowInputFormatWithDiagnosticInfo.h @@ -14,7 +14,9 @@ class RowInputFormatWithDiagnosticInfo : public IRowInputFormat public: RowInputFormatWithDiagnosticInfo(const Block & header_, ReadBuffer & in_, const Params & params_); + std::pair getDiagnosticAndRawDataImpl(bool is_errors_record); String getDiagnosticInfo() override; + std::pair getDiagnosticAndRawData() override; void resetParser() override; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 60bf8d6a15c..91777f7d642 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -143,7 +143,12 @@ ReadFromMergeTree::ReadFromMergeTree( { auto const & settings = context->getSettingsRef(); if ((settings.optimize_read_in_order || settings.optimize_aggregation_in_order) && query_info.getInputOrderInfo()) + { output_stream->sort_scope = DataStream::SortScope::Stream; + const size_t used_prefix_of_sorting_key_size = query_info.getInputOrderInfo()->used_prefix_of_sorting_key_size; + if (sort_description.size() > used_prefix_of_sorting_key_size) + sort_description.resize(used_prefix_of_sorting_key_size); + } else output_stream->sort_scope = DataStream::SortScope::Chunk; } diff --git a/src/Processors/Sources/ShellCommandSource.h b/src/Processors/Sources/ShellCommandSource.h index a0b4aff4c1b..80ba1d59adf 100644 --- a/src/Processors/Sources/ShellCommandSource.h +++ b/src/Processors/Sources/ShellCommandSource.h @@ -58,7 +58,7 @@ public: /// Pool size valid only if executable_pool = true size_t pool_size = 16; - /// Max command execution time in milliseconds. Valid only if executable_pool = true + /// Max command execution time in seconds. Valid only if executable_pool = true size_t max_command_execution_time_seconds = 10; /// Should pool of processes be created. diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 4e55081ca48..ca8e9c0c85b 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -248,8 +248,11 @@ private: throw Exception(ErrorCodes::LOGICAL_ERROR, "Some ready chunks expected"); auto & output = outputs.front(); - output.push(std::move(single_level_chunks.back())); + auto chunk = std::move(single_level_chunks.back()); single_level_chunks.pop_back(); + const auto has_rows = chunk.hasRows(); + if (has_rows) + output.push(std::move(chunk)); if (finished && single_level_chunks.empty()) { @@ -257,7 +260,7 @@ private: return Status::Finished; } - return Status::PortFull; + return has_rows ? Status::PortFull : Status::Ready; } /// Read all sources and try to push current bucket. @@ -281,7 +284,10 @@ private: if (!two_level_chunks[current_bucket_num]) return Status::NeedData; - output.push(std::move(two_level_chunks[current_bucket_num])); + auto chunk = std::move(two_level_chunks[current_bucket_num]); + const auto has_rows = chunk.hasRows(); + if (has_rows) + output.push(std::move(chunk)); ++current_bucket_num; if (current_bucket_num == NUM_BUCKETS) @@ -291,7 +297,7 @@ private: return Status::Finished; } - return Status::PortFull; + return has_rows ? Status::PortFull : Status::Ready; } AggregatingTransformParamsPtr params; diff --git a/src/Processors/Transforms/MongoDBSource.cpp b/src/Processors/Transforms/MongoDBSource.cpp index 19d21f3409e..19289f3f818 100644 --- a/src/Processors/Transforms/MongoDBSource.cpp +++ b/src/Processors/Transforms/MongoDBSource.cpp @@ -250,13 +250,13 @@ namespace if (value.type() == Poco::MongoDB::ElementTraits::TypeId) { std::string string_id = value.toString(); - assert_cast(column).insertDataWithTerminatingZero(string_id.data(), string_id.size() + 1); + assert_cast(column).insertData(string_id.data(), string_id.size()); break; } else if (value.type() == Poco::MongoDB::ElementTraits::TypeId) { String string = static_cast &>(value).value(); - assert_cast(column).insertDataWithTerminatingZero(string.data(), string.size() + 1); + assert_cast(column).insertData(string.data(), string.size()); break; } diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 16981d26146..e8d48431a9e 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -140,6 +140,11 @@ namespace size_t rows = 0; size_t bytes = 0; + UInt32 shard_num = 0; + std::string cluster; + std::string distributed_table; + std::string remote_table; + /// dumpStructure() of the header -- obsolete std::string block_header_string; Block block_header; @@ -195,6 +200,14 @@ namespace in.getFileName(), distributed_header.revision, DBMS_TCP_PROTOCOL_VERSION); } + if (header_buf.hasPendingData()) + { + readVarUInt(distributed_header.shard_num, header_buf); + readStringBinary(distributed_header.cluster, header_buf); + readStringBinary(distributed_header.distributed_table, header_buf); + readStringBinary(distributed_header.remote_table, header_buf); + } + /// Add handling new data here, for example: /// /// if (header_buf.hasPendingData()) @@ -621,18 +634,23 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa ReadBufferFromFile in(file_path); const auto & distributed_header = readDistributedHeader(in, log); - auto connection = pool->get(timeouts, &distributed_header.insert_settings); + thread_trace_context = std::make_unique(__PRETTY_FUNCTION__, + distributed_header.client_info.client_trace_context, + this->storage.getContext()->getOpenTelemetrySpanLog()); + thread_trace_context->root_span.addAttribute("clickhouse.shard_num", distributed_header.shard_num); + thread_trace_context->root_span.addAttribute("clickhouse.cluster", distributed_header.cluster); + thread_trace_context->root_span.addAttribute("clickhouse.distributed", distributed_header.distributed_table); + thread_trace_context->root_span.addAttribute("clickhouse.remote", distributed_header.remote_table); + thread_trace_context->root_span.addAttribute("clickhouse.rows", distributed_header.rows); + thread_trace_context->root_span.addAttribute("clickhouse.bytes", distributed_header.bytes); + auto connection = pool->get(timeouts, &distributed_header.insert_settings); LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)", file_path, connection->getDescription(), formatReadableQuantity(distributed_header.rows), formatReadableSizeWithBinarySuffix(distributed_header.bytes)); - thread_trace_context = std::make_unique(__PRETTY_FUNCTION__, - distributed_header.client_info.client_trace_context, - this->storage.getContext()->getOpenTelemetrySpanLog()); - RemoteInserter remote{*connection, timeouts, distributed_header.insert_query, distributed_header.insert_settings, diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index ae72fdd84e2..8099a7f2002 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -171,7 +171,6 @@ void DistributedSink::writeAsync(const Block & block) } else { - if (storage.getShardingKeyExpr() && (cluster->getShardsInfo().size() > 1)) return writeSplitAsync(block); @@ -291,6 +290,8 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si auto thread_group = CurrentThread::getGroup(); return [this, thread_group, &job, ¤t_block, num_shards]() { + OpenTelemetry::SpanHolder span(__PRETTY_FUNCTION__); + if (thread_group) CurrentThread::attachToIfDetached(thread_group); setThreadName("DistrOutStrProc"); @@ -331,15 +332,19 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si const Block & shard_block = (num_shards > 1) ? job.current_shard_block : current_block; const Settings & settings = context->getSettingsRef(); - /// Do not initiate INSERT for empty block. size_t rows = shard_block.rows(); + + span.addAttribute("clickhouse.shard_num", shard_info.shard_num); + span.addAttribute("clickhouse.cluster", this->storage.cluster_name); + span.addAttribute("clickhouse.distributed", this->storage.getStorageID().getFullNameNotQuoted()); + span.addAttribute("clickhouse.remote", [this]() { return storage.remote_database + "." + storage.remote_table; }); + span.addAttribute("clickhouse.rows", rows); + span.addAttribute("clickhouse.bytes", [&shard_block]() { return toString(shard_block.bytes()); }); + + /// Do not initiate INSERT for empty block. if (rows == 0) return; - OpenTelemetry::SpanHolder span(__PRETTY_FUNCTION__); - span.addAttribute("clickhouse.shard_num", shard_info.shard_num); - span.addAttribute("clickhouse.written_rows", rows); - if (!job.is_local_job || !settings.prefer_localhost_replica) { if (!job.executor) @@ -610,20 +615,15 @@ void DistributedSink::writeSplitAsync(const Block & block) void DistributedSink::writeAsyncImpl(const Block & block, size_t shard_id) { - OpenTelemetry::SpanHolder span("DistributedSink::writeAsyncImpl()"); - const auto & shard_info = cluster->getShardsInfo()[shard_id]; const auto & settings = context->getSettingsRef(); Block block_to_send = removeSuperfluousColumns(block); - span.addAttribute("clickhouse.shard_num", shard_info.shard_num); - span.addAttribute("clickhouse.written_rows", block.rows()); - if (shard_info.hasInternalReplication()) { if (shard_info.isLocal() && settings.prefer_localhost_replica) /// Prefer insert into current instance directly - writeToLocal(block_to_send, shard_info.getLocalNodeCount()); + writeToLocal(shard_info, block_to_send, shard_info.getLocalNodeCount()); else { const auto & path = shard_info.insertPathForInternalReplication( @@ -631,13 +631,13 @@ void DistributedSink::writeAsyncImpl(const Block & block, size_t shard_id) settings.use_compact_format_in_distributed_parts_names); if (path.empty()) throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR); - writeToShard(block_to_send, {path}); + writeToShard(shard_info, block_to_send, {path}); } } else { if (shard_info.isLocal() && settings.prefer_localhost_replica) - writeToLocal(block_to_send, shard_info.getLocalNodeCount()); + writeToLocal(shard_info, block_to_send, shard_info.getLocalNodeCount()); std::vector dir_names; for (const auto & address : cluster->getShardsAddresses()[shard_id]) @@ -645,30 +645,44 @@ void DistributedSink::writeAsyncImpl(const Block & block, size_t shard_id) dir_names.push_back(address.toFullString(settings.use_compact_format_in_distributed_parts_names)); if (!dir_names.empty()) - writeToShard(block_to_send, dir_names); + writeToShard(shard_info, block_to_send, dir_names); } } -void DistributedSink::writeToLocal(const Block & block, size_t repeats) +void DistributedSink::writeToLocal(const Cluster::ShardInfo & shard_info, const Block & block, size_t repeats) { OpenTelemetry::SpanHolder span(__PRETTY_FUNCTION__); - span.addAttribute("db.statement", this->query_string); + span.addAttribute("clickhouse.shard_num", shard_info.shard_num); + span.addAttribute("clickhouse.cluster", this->storage.cluster_name); + span.addAttribute("clickhouse.distributed", this->storage.getStorageID().getFullNameNotQuoted()); + span.addAttribute("clickhouse.remote", [this]() { return storage.remote_database + "." + storage.remote_table; }); + span.addAttribute("clickhouse.rows", [&block]() { return toString(block.rows()); }); + span.addAttribute("clickhouse.bytes", [&block]() { return toString(block.bytes()); }); - InterpreterInsertQuery interp(query_ast, context, allow_materialized); + try + { + InterpreterInsertQuery interp(query_ast, context, allow_materialized); - auto block_io = interp.execute(); - PushingPipelineExecutor executor(block_io.pipeline); + auto block_io = interp.execute(); + PushingPipelineExecutor executor(block_io.pipeline); - executor.start(); - writeBlockConvert(executor, block, repeats, log); - executor.finish(); + executor.start(); + writeBlockConvert(executor, block, repeats, log); + executor.finish(); + } + catch (...) + { + span.addAttribute(std::current_exception()); + throw; + } } -void DistributedSink::writeToShard(const Block & block, const std::vector & dir_names) +void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const Block & block, const std::vector & dir_names) { OpenTelemetry::SpanHolder span(__PRETTY_FUNCTION__); + span.addAttribute("clickhouse.shard_num", shard_info.shard_num); const auto & settings = context->getSettingsRef(); const auto & distributed_settings = storage.getDistributedSettingsRef(); @@ -759,6 +773,11 @@ void DistributedSink::writeToShard(const Block & block, const std::vectorstorage.cluster_name, header_buf); + writeStringBinary(this->storage.getStorageID().getFullNameNotQuoted(), header_buf); + writeStringBinary(this->storage.remote_database + "." + this->storage.remote_table, header_buf); + /// Add new fields here, for example: /// writeVarUInt(my_new_data, header_buf); /// And note that it is safe, because we have checksum and size for header. diff --git a/src/Storages/Distributed/DistributedSink.h b/src/Storages/Distributed/DistributedSink.h index 668cec22e8b..af0c64cbd78 100644 --- a/src/Storages/Distributed/DistributedSink.h +++ b/src/Storages/Distributed/DistributedSink.h @@ -69,9 +69,9 @@ private: Block removeSuperfluousColumns(Block block) const; /// Increments finished_writings_count after each repeat. - void writeToLocal(const Block & block, size_t repeats); + void writeToLocal(const Cluster::ShardInfo & shard_info, const Block & block, size_t repeats); - void writeToShard(const Block & block, const std::vector & dir_names); + void writeToShard(const Cluster::ShardInfo & shard_info, const Block & block, const std::vector & dir_names); /// Performs synchronous insertion to remote nodes. If timeout_exceeded flag was set, throws. diff --git a/src/Storages/Freeze.cpp b/src/Storages/Freeze.cpp index 5a3056450a0..a2f0395b001 100644 --- a/src/Storages/Freeze.cpp +++ b/src/Storages/Freeze.cpp @@ -5,12 +5,29 @@ #include #include +/** + * When ClickHouse has frozen data on remote storage it required 'smart' data removing during UNFREEZE. + * For remote storage actually frozen not remote data but local metadata with referrers on remote data. + * So remote data can be referred from working and frozen data sets (or two frozen) at same time. + * In this case during UNFREEZE ClickHouse should remove only local metadata and keep remote data. + * But when data was already removed from working data set ClickHouse should remove remote data too. + * To detect is current data used or not in some other place ClickHouse uses + * - ref_count from metadata to check if data used in some other metadata on the same replica; + * - Keeper record to check if data used on other replica. + * StorageReplicatedMergeTree::removeSharedDetachedPart makes required checks, so here this method + * called for each frozen part. + */ + namespace DB { + +namespace ErrorCodes +{ + extern const int SUPPORT_IS_DISABLED; +} + void FreezeMetaData::fill(const StorageReplicatedMergeTree & storage) { - is_replicated = storage.supportsReplication(); - is_remote = storage.isRemote(); replica_name = storage.getReplicaName(); zookeeper_name = storage.getZooKeeperName(); table_shared_id = storage.getTableSharedID(); @@ -26,11 +43,17 @@ void FreezeMetaData::save(DiskPtr data_disk, const String & path) const writeIntText(version, buffer); buffer.write("\n", 1); - writeBoolText(is_replicated, buffer); - buffer.write("\n", 1); - writeBoolText(is_remote, buffer); - buffer.write("\n", 1); - writeString(replica_name, buffer); + if (version == 1) + { + /// is_replicated and is_remote are not used + bool is_replicated = true; + writeBoolText(is_replicated, buffer); + buffer.write("\n", 1); + bool is_remote = true; + writeBoolText(is_remote, buffer); + buffer.write("\n", 1); + } + writeString(escapeForFileName(replica_name), buffer); buffer.write("\n", 1); writeString(zookeeper_name, buffer); buffer.write("\n", 1); @@ -51,17 +74,25 @@ bool FreezeMetaData::load(DiskPtr data_disk, const String & path) auto metadata_str = metadata_storage->readFileToString(file_path); ReadBufferFromString buffer(metadata_str); readIntText(version, buffer); - if (version != 1) + if (version < 1 || version > 2) { - LOG_ERROR(&Poco::Logger::get("FreezeMetaData"), "Unknown freezed metadata version: {}", version); + LOG_ERROR(&Poco::Logger::get("FreezeMetaData"), "Unknown frozen metadata version: {}", version); return false; } DB::assertChar('\n', buffer); - readBoolText(is_replicated, buffer); - DB::assertChar('\n', buffer); - readBoolText(is_remote, buffer); - DB::assertChar('\n', buffer); - readString(replica_name, buffer); + if (version == 1) + { + /// is_replicated and is_remote are not used + bool is_replicated; + readBoolText(is_replicated, buffer); + DB::assertChar('\n', buffer); + bool is_remote; + readBoolText(is_remote, buffer); + DB::assertChar('\n', buffer); + } + std::string unescaped_replica_name; + readString(unescaped_replica_name, buffer); + replica_name = unescapeForFileName(unescaped_replica_name); DB::assertChar('\n', buffer); readString(zookeeper_name, buffer); DB::assertChar('\n', buffer); @@ -87,9 +118,23 @@ String FreezeMetaData::getFileName(const String & path) return fs::path(path) / "frozen_metadata.txt"; } -BlockIO Unfreezer::unfreeze(const String & backup_name, ContextPtr local_context) +Unfreezer::Unfreezer(ContextPtr context) : local_context(context) { - LOG_DEBUG(log, "Unfreezing backup {}", backup_name); + if (local_context->hasZooKeeper()) + zookeeper = local_context->getZooKeeper(); +} + +BlockIO Unfreezer::systemUnfreeze(const String & backup_name) +{ + LOG_DEBUG(log, "Unfreezing backup {}", escapeForFileName(backup_name)); + + const auto & config = local_context->getConfigRef(); + static constexpr auto config_key = "enable_system_unfreeze"; + if (!config.getBool(config_key, false)) + { + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Support for SYSTEM UNFREEZE query is disabled. You can enable it via '{}' server setting", config_key); + } + auto disks_map = local_context->getDisksMap(); Disks disks; for (auto & [name, disk]: disks_map) @@ -97,33 +142,38 @@ BlockIO Unfreezer::unfreeze(const String & backup_name, ContextPtr local_context disks.push_back(disk); } auto backup_path = fs::path(backup_directory_prefix) / escapeForFileName(backup_name); - auto store_path = backup_path / "store"; + auto store_paths = {backup_path / "store", backup_path / "data"}; PartitionCommandsResultInfo result_info; for (const auto & disk: disks) { - if (!disk->exists(store_path)) - continue; - for (auto prefix_it = disk->iterateDirectory(store_path); prefix_it->isValid(); prefix_it->next()) + for (const auto& store_path: store_paths) { - auto prefix_directory = store_path / prefix_it->name(); - for (auto table_it = disk->iterateDirectory(prefix_directory); table_it->isValid(); table_it->next()) + if (!disk->exists(store_path)) + continue; + for (auto prefix_it = disk->iterateDirectory(store_path); prefix_it->isValid(); prefix_it->next()) { - auto table_directory = prefix_directory / table_it->name(); - auto current_result_info = unfreezePartitionsFromTableDirectory([] (const String &) { return true; }, backup_name, {disk}, table_directory, local_context); - for (auto & command_result : current_result_info) + auto prefix_directory = store_path / prefix_it->name(); + for (auto table_it = disk->iterateDirectory(prefix_directory); table_it->isValid(); table_it->next()) { - command_result.command_type = "SYSTEM UNFREEZE"; + auto table_directory = prefix_directory / table_it->name(); + auto current_result_info = unfreezePartitionsFromTableDirectory( + [](const String &) { return true; }, backup_name, {disk}, table_directory); + for (auto & command_result : current_result_info) + { + command_result.command_type = "SYSTEM UNFREEZE"; + } + result_info.insert( + result_info.end(), + std::make_move_iterator(current_result_info.begin()), + std::make_move_iterator(current_result_info.end())); } - result_info.insert( - result_info.end(), - std::make_move_iterator(current_result_info.begin()), - std::make_move_iterator(current_result_info.end())); } } if (disk->exists(backup_path)) { + /// After unfreezing we need to clear revision.txt file and empty directories disk->removeRecursive(backup_path); } } @@ -136,18 +186,15 @@ BlockIO Unfreezer::unfreeze(const String & backup_name, ContextPtr local_context return result; } -bool Unfreezer::removeFreezedPart(DiskPtr disk, const String & path, const String & part_name, ContextPtr local_context) +bool Unfreezer::removeFreezedPart(DiskPtr disk, const String & path, const String & part_name, ContextPtr local_context, zkutil::ZooKeeperPtr zookeeper) { if (disk->supportZeroCopyReplication()) { FreezeMetaData meta; if (meta.load(disk, path)) { - if (meta.is_replicated) - { - FreezeMetaData::clean(disk, path); - return StorageReplicatedMergeTree::removeSharedDetachedPart(disk, path, part_name, meta.table_shared_id, meta.zookeeper_name, meta.replica_name, "", local_context); - } + FreezeMetaData::clean(disk, path); + return StorageReplicatedMergeTree::removeSharedDetachedPart(disk, path, part_name, meta.table_shared_id, meta.zookeeper_name, meta.replica_name, "", local_context, zookeeper); } } @@ -156,7 +203,7 @@ bool Unfreezer::removeFreezedPart(DiskPtr disk, const String & path, const Strin return false; } -PartitionCommandsResultInfo Unfreezer::unfreezePartitionsFromTableDirectory(MergeTreeData::MatcherFn matcher, const String & backup_name, const Disks & disks, const fs::path & table_directory, ContextPtr local_context) +PartitionCommandsResultInfo Unfreezer::unfreezePartitionsFromTableDirectory(MergeTreeData::MatcherFn matcher, const String & backup_name, const Disks & disks, const fs::path & table_directory) { PartitionCommandsResultInfo result; @@ -180,7 +227,7 @@ PartitionCommandsResultInfo Unfreezer::unfreezePartitionsFromTableDirectory(Merg const auto & path = it->path(); - bool keep_shared = removeFreezedPart(disk, path, partition_directory, local_context); + bool keep_shared = removeFreezedPart(disk, path, partition_directory, local_context, zookeeper); result.push_back(PartitionCommandResultInfo{ .partition_id = partition_id, diff --git a/src/Storages/Freeze.h b/src/Storages/Freeze.h index 290121cfc1e..a64be7465dd 100644 --- a/src/Storages/Freeze.h +++ b/src/Storages/Freeze.h @@ -23,9 +23,7 @@ private: static String getFileName(const String & path); public: - int version = 1; - bool is_replicated{false}; - bool is_remote{false}; + int version = 2; String replica_name; String zookeeper_name; String table_shared_id; @@ -34,12 +32,15 @@ public: class Unfreezer { public: - PartitionCommandsResultInfo unfreezePartitionsFromTableDirectory(MergeTreeData::MatcherFn matcher, const String & backup_name, const Disks & disks, const fs::path & table_directory, ContextPtr local_context); - BlockIO unfreeze(const String & backup_name, ContextPtr local_context); + Unfreezer(ContextPtr context); + PartitionCommandsResultInfo unfreezePartitionsFromTableDirectory(MergeTreeData::MatcherFn matcher, const String & backup_name, const Disks & disks, const fs::path & table_directory); + BlockIO systemUnfreeze(const String & backup_name); private: + ContextPtr local_context; + zkutil::ZooKeeperPtr zookeeper; Poco::Logger * log = &Poco::Logger::get("Unfreezer"); static constexpr std::string_view backup_directory_prefix = "shadow"; - static bool removeFreezedPart(DiskPtr disk, const String & path, const String & part_name, ContextPtr local_context); + static bool removeFreezedPart(DiskPtr disk, const String & path, const String & part_name, ContextPtr local_context, zkutil::ZooKeeperPtr zookeeper); }; } diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index f93bc45d1a3..45caddb21ea 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -255,7 +255,7 @@ private: class HDFSSource::URISIterator::Impl { public: - explicit Impl(const std::vector & uris_, ContextPtr context) + explicit Impl(const std::vector & uris_, ContextPtr context) { auto path_and_uri = getPathFromUriAndUriWithoutPath(uris_[0]); HDFSBuilderWrapper builder = createHDFSBuilder(path_and_uri.second + "/", context->getGlobalContext()->getConfigRef()); @@ -293,7 +293,7 @@ String HDFSSource::DisclosedGlobIterator::next() return pimpl->next(); } -HDFSSource::URISIterator::URISIterator(const std::vector & uris_, ContextPtr context) +HDFSSource::URISIterator::URISIterator(const std::vector & uris_, ContextPtr context) : pimpl(std::make_shared(uris_, context)) { } diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index a0d61f4bd2a..896371f9685 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -86,7 +86,7 @@ private: const String & format_name, const ContextPtr & ctx); - std::vector uris; + std::vector uris; String format_name; String compression_method; const bool distributed_processing; @@ -116,7 +116,7 @@ public: class URISIterator { public: - URISIterator(const std::vector & uris_, ContextPtr context); + URISIterator(const std::vector & uris_, ContextPtr context); String next(); private: class Impl; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 06ce4fb308d..d9bacffd053 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -179,8 +179,6 @@ namespace void loadFromConfig(cppkafka::Configuration & conf, const Poco::Util::AbstractConfiguration & config, const std::string & path) { Poco::Util::AbstractConfiguration::Keys keys; - std::vector errstr(512); - config.keys(path, keys); for (const auto & key : keys) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 894eec12f0c..5245bc89e0c 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -204,13 +204,12 @@ DataPartStorageBuilderPtr DataPartStorageOnDisk::getBuilder() const } void DataPartStorageOnDisk::remove( - bool can_remove_shared_data, - const NameSet & names_not_to_remove, + CanRemoveCallback && can_remove_callback, const MergeTreeDataPartChecksums & checksums, std::list projections, bool is_temp, MergeTreeDataPartState state, - Poco::Logger * log) const + Poco::Logger * log) { /// NOTE We rename part to delete_tmp_ instead of delete_tmp_ to avoid race condition /// when we try to remove two parts with the same name, but different relative paths, @@ -239,13 +238,16 @@ void DataPartStorageOnDisk::remove( fs::path to = fs::path(root_path) / part_dir_without_slash; + std::optional can_remove_description; + auto disk = volume->getDisk(); if (disk->exists(to)) { LOG_WARNING(log, "Directory {} (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart or race condition. Removing it.", fullPath(disk, to)); try { - disk->removeSharedRecursive(fs::path(to) / "", !can_remove_shared_data, names_not_to_remove); + can_remove_description.emplace(can_remove_callback()); + disk->removeSharedRecursive(fs::path(to) / "", !can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove); } catch (...) { @@ -257,6 +259,7 @@ void DataPartStorageOnDisk::remove( try { disk->moveDirectory(from, to); + onRename(root_path, part_dir_without_slash); } catch (const fs::filesystem_error & e) { @@ -268,6 +271,9 @@ void DataPartStorageOnDisk::remove( throw; } + if (!can_remove_description) + can_remove_description.emplace(can_remove_callback()); + // Record existing projection directories so we don't remove them twice std::unordered_set projection_directories; std::string proj_suffix = ".proj"; @@ -278,7 +284,7 @@ void DataPartStorageOnDisk::remove( clearDirectory( fs::path(to) / proj_dir_name, - can_remove_shared_data, names_not_to_remove, projection.checksums, {}, is_temp, state, log, true); + can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove, projection.checksums, {}, is_temp, state, log, true); } /// It is possible that we are removing the part which have a written but not loaded projection. @@ -305,7 +311,7 @@ void DataPartStorageOnDisk::remove( clearDirectory( fs::path(to) / name, - can_remove_shared_data, names_not_to_remove, tmp_checksums, {}, is_temp, state, log, true); + can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove, tmp_checksums, {}, is_temp, state, log, true); } catch (...) { @@ -315,7 +321,7 @@ void DataPartStorageOnDisk::remove( } } - clearDirectory(to, can_remove_shared_data, names_not_to_remove, checksums, projection_directories, is_temp, state, log, false); + clearDirectory(to, can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove, checksums, projection_directories, is_temp, state, log, false); } void DataPartStorageOnDisk::clearDirectory( @@ -348,18 +354,11 @@ void DataPartStorageOnDisk::clearDirectory( /// Remove each expected file in directory, then remove directory itself. RemoveBatchRequest request; -#if !defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-variable" -#endif for (const auto & [file, _] : checksums.files) { if (skip_directories.find(file) == skip_directories.end()) request.emplace_back(fs::path(dir) / file); } -#if !defined(__clang__) -# pragma GCC diagnostic pop -#endif for (const auto & file : {"checksums.txt", "columns.txt"}) request.emplace_back(fs::path(dir) / file); diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index f02ef26f811..51b557767d4 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -45,13 +45,12 @@ public: void checkConsistency(const MergeTreeDataPartChecksums & checksums) const override; void remove( - bool can_remove_shared_data, - const NameSet & names_not_to_remove, + CanRemoveCallback && can_remove_callback, const MergeTreeDataPartChecksums & checksums, std::list projections, bool is_temp, MergeTreeDataPartState state, - Poco::Logger * log) const override; + Poco::Logger * log) override; std::string getRelativePathForPrefix(Poco::Logger * log, const String & prefix, bool detached) const override; diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 9da8a5eae03..bd449d46075 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -12,6 +12,13 @@ namespace DB class ReadBufferFromFileBase; class WriteBufferFromFileBase; +struct CanRemoveDescription +{ + bool can_remove_anything; + NameSet files_not_to_remove; + +}; +using CanRemoveCallback = std::function; class IDataPartStorageIterator { @@ -113,13 +120,12 @@ public: /// can_remove_shared_data, names_not_to_remove are specific for DiskObjectStorage. /// projections, checksums are needed to avoid recursive listing virtual void remove( - bool can_remove_shared_data, - const NameSet & names_not_to_remove, + CanRemoveCallback && can_remove_callback, const MergeTreeDataPartChecksums & checksums, std::list projections, bool is_temp, MergeTreeDataPartState state, - Poco::Logger * log) const = 0; + Poco::Logger * log) = 0; /// Get a name like 'prefix_partdir_tryN' which does not exist in a root dir. /// TODO: remove it. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index e9d900c6d54..217f437a4ff 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1291,7 +1291,7 @@ catch (Exception & e) bool IMergeTreeDataPart::wasInvolvedInTransaction() const { - assert(!version.creation_tid.isEmpty() || (state == MergeTreeDataPartState::Temporary /* && std::uncaught_exceptions() */)); + assert(!storage.data_parts_loading_finished || !version.creation_tid.isEmpty() || (state == MergeTreeDataPartState::Temporary /* && std::uncaught_exceptions() */)); bool created_by_transaction = !version.creation_tid.isPrehistoric(); bool removed_by_transaction = version.isRemovalTIDLocked() && version.removal_tid_lock != Tx::PrehistoricTID.getHash(); return created_by_transaction || removed_by_transaction; @@ -1433,13 +1433,18 @@ void IMergeTreeDataPart::remove() const assert(assertHasValidVersionMetadata()); part_is_probably_removed_from_disk = true; - auto [can_remove, files_not_to_remove] = canRemovePart(); + auto can_remove_callback = [this] () + { + auto [can_remove, files_not_to_remove] = canRemovePart(); + if (!can_remove) + LOG_TRACE(storage.log, "Blobs of part {} cannot be removed", name); - if (!can_remove) - LOG_TRACE(storage.log, "Blobs of part {} cannot be removed", name); + if (!files_not_to_remove.empty()) + LOG_TRACE(storage.log, "Some blobs ({}) of part {} cannot be removed", fmt::join(files_not_to_remove, ", "), name); + + return CanRemoveDescription{.can_remove_anything = can_remove, .files_not_to_remove = files_not_to_remove }; + }; - if (!files_not_to_remove.empty()) - LOG_TRACE(storage.log, "Some blobs ({}) of part {} cannot be removed", fmt::join(files_not_to_remove, ", "), name); if (!isStoredOnDisk()) return; @@ -1459,7 +1464,7 @@ void IMergeTreeDataPart::remove() const projection_checksums.emplace_back(IDataPartStorage::ProjectionChecksums{.name = p_name, .checksums = projection_part->checksums}); } - data_part_storage->remove(can_remove, files_not_to_remove, checksums, projection_checksums, is_temp, getState(), storage.log); + data_part_storage->remove(std::move(can_remove_callback), checksums, projection_checksums, is_temp, getState(), storage.log); } String IMergeTreeDataPart::getRelativePathForPrefix(const String & prefix, bool detached) const diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index f5eeb4ed35c..2eea7a2f359 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -116,10 +116,20 @@ static void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuf { switch (node.type) { - case (ActionsDAG::ActionType::INPUT): [[fallthrough]]; - case (ActionsDAG::ActionType::COLUMN): + case (ActionsDAG::ActionType::INPUT): writeString(node.result_name, out); break; + case (ActionsDAG::ActionType::COLUMN): + { + /// If it was created from ASTLiteral, then result_name can be an alias. + /// We need to convert value back to string here. + if (const auto * column_const = typeid_cast(node.column.get())) + writeString(applyVisitor(FieldVisitorToString(), column_const->getField()), out); + /// It may be possible that column is ColumnSet + else + writeString(node.result_name, out); + break; + } case (ActionsDAG::ActionType::ALIAS): appendColumnNameWithoutAlias(*node.children.front(), out, legacy); break; diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index d51cd6aa07d..9bae4a840bb 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -223,6 +223,27 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() .part_log_writer = {} }; } + else if (!storage.findReplicaHavingCoveringPart(entry.new_part_name, /* active */ false, dummy).empty()) + { + /// Why this if still needed? We can check for part in zookeeper, don't find it and sleep for any amount of time. During this sleep part will be actually committed from other replica + /// and exclusive zero copy lock will be released. We will take the lock and execute merge one more time, while it was possible just to download the part from other replica. + /// + /// It's also possible just because reads in [Zoo]Keeper are not lineariazable. + /// + /// NOTE: In case of mutation and hardlinks it can even lead to extremely rare dataloss (we will produce new part with the same hardlinks, don't fetch the same from other replica), so this check is important. + zero_copy_lock->lock->unlock(); + + LOG_DEBUG(log, "We took zero copy lock, but merge of part {} finished by some other replica, will release lock and download merged part to avoid data duplication", entry.new_part_name); + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; + } + else + { + LOG_DEBUG(log, "Zero copy lock taken, will merge part {}", entry.new_part_name); + } } } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 24fb5b292c2..23701c36814 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -153,6 +154,7 @@ namespace ErrorCodes extern const int TOO_MANY_SIMULTANEOUS_QUERIES; extern const int INCORRECT_QUERY; extern const int CANNOT_RESTORE_TABLE; + extern const int ZERO_COPY_REPLICATION_ERROR; } static void checkSampleExpression(const StorageInMemoryMetadata & metadata, bool allow_sampling_expression_not_in_primary_key, bool check_sample_column_is_correct) @@ -1560,6 +1562,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) calculateColumnAndSecondaryIndexSizesImpl(); LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size()); + data_parts_loading_finished = true; } /// Is the part directory old. @@ -1634,11 +1637,10 @@ size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lif /// We don't control the amount of refs for temporary parts so we cannot decide can we remove blobs /// or not. So we are not doing it bool keep_shared = false; - if (it->path().find("fetch") != std::string::npos) + if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication) { - keep_shared = disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication; - if (keep_shared) - LOG_WARNING(log, "Since zero-copy replication is enabled we are not going to remove blobs from shared storage for {}", full_path); + LOG_WARNING(log, "Since zero-copy replication is enabled we are not going to remove blobs from shared storage for {}", full_path); + keep_shared = true; } disk->removeSharedRecursive(it->path(), keep_shared, {}); @@ -1992,7 +1994,7 @@ size_t MergeTreeData::clearOldBrokenPartsFromDetachedDirectory() for (auto & [old_name, new_name, disk] : renamed_parts.old_and_new_names) { - removeDetachedPart(disk, fs::path(relative_data_path) / "detached" / new_name / "", old_name, false); + removeDetachedPart(disk, fs::path(relative_data_path) / "detached" / new_name / "", old_name); LOG_DEBUG(log, "Removed broken detached part {} due to a timeout for broken detached parts", old_name); old_name.clear(); } @@ -2134,6 +2136,7 @@ void MergeTreeData::renameInMemory(const StorageID & new_table_id) void MergeTreeData::dropAllData() { LOG_TRACE(log, "dropAllData: waiting for locks."); + auto settings_ptr = getSettings(); auto lock = lockParts(); @@ -2176,6 +2179,9 @@ void MergeTreeData::dropAllData() throw; } + LOG_INFO(log, "dropAllData: clearing temporary directories"); + clearOldTemporaryDirectories(0, {"tmp_", "delete_tmp_", "tmp-fetch_"}); + column_sizes.clear(); for (const auto & disk : getDisks()) @@ -2183,8 +2189,29 @@ void MergeTreeData::dropAllData() if (disk->isBroken()) continue; + LOG_INFO(log, "dropAllData: remove format_version.txt, detached, moving and write ahead logs"); + disk->removeFileIfExists(fs::path(relative_data_path) / FORMAT_VERSION_FILE_NAME); + + if (disk->exists(fs::path(relative_data_path) / DETACHED_DIR_NAME)) + disk->removeRecursive(fs::path(relative_data_path) / DETACHED_DIR_NAME); + + if (disk->exists(fs::path(relative_data_path) / MOVING_DIR_NAME)) + disk->removeRecursive(fs::path(relative_data_path) / MOVING_DIR_NAME); + + MergeTreeWriteAheadLog::dropAllWriteAheadLogs(disk, relative_data_path); + try { + if (!disk->isDirectoryEmpty(relative_data_path) && supportsReplication() && disk->supportZeroCopyReplication() && settings_ptr->allow_remote_fs_zero_copy_replication) + { + std::vector files_left; + disk->listFiles(relative_data_path, files_left); + + throw Exception( + ErrorCodes::ZERO_COPY_REPLICATION_ERROR, "Directory {} with table {} not empty (files [{}]) after drop. Will not drop.", + relative_data_path, getStorageID().getNameForLogs(), fmt::join(files_left, ", ")); + } + LOG_INFO(log, "dropAllData: removing table directory recursive to cleanup garbage"); disk->removeRecursive(relative_data_path); } @@ -4744,7 +4771,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr for (auto & [old_name, new_name, disk] : renamed_parts.old_and_new_names) { - bool keep_shared = removeDetachedPart(disk, fs::path(relative_data_path) / "detached" / new_name / "", old_name, false); + bool keep_shared = removeDetachedPart(disk, fs::path(relative_data_path) / "detached" / new_name / "", old_name); LOG_DEBUG(log, "Dropped detached part {}, keep shared data: {}", old_name, keep_shared); old_name.clear(); } @@ -6411,7 +6438,7 @@ PartitionCommandsResultInfo MergeTreeData::unfreezeAll( return unfreezePartitionsByMatcher([] (const String &) { return true; }, backup_name, local_context); } -bool MergeTreeData::removeDetachedPart(DiskPtr disk, const String & path, const String &, bool) +bool MergeTreeData::removeDetachedPart(DiskPtr disk, const String & path, const String &) { disk->removeRecursive(path); @@ -6426,7 +6453,7 @@ PartitionCommandsResultInfo MergeTreeData::unfreezePartitionsByMatcher(MatcherFn auto disks = getStoragePolicy()->getDisks(); - return Unfreezer().unfreezePartitionsFromTableDirectory(matcher, backup_name, disks, backup_path, local_context); + return Unfreezer(local_context).unfreezePartitionsFromTableDirectory(matcher, backup_name, disks, backup_path); } bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 3a35daf4c90..2cff9048898 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -151,6 +151,7 @@ public: constexpr static auto FORMAT_VERSION_FILE_NAME = "format_version.txt"; constexpr static auto DETACHED_DIR_NAME = "detached"; + constexpr static auto MOVING_DIR_NAME = "moving"; /// Auxiliary structure for index comparison. Keep in mind lifetime of MergeTreePartInfo. struct DataPartStateAndInfo @@ -970,7 +971,7 @@ public: /// Check shared data usage on other replicas for detached/freezed part /// Remove local files and remote files if needed - virtual bool removeDetachedPart(DiskPtr disk, const String & path, const String & part_name, bool is_freezed); + virtual bool removeDetachedPart(DiskPtr disk, const String & path, const String & part_name); virtual String getTableSharedID() const { return ""; } @@ -1034,6 +1035,8 @@ protected: /// True if at least one part was created/removed with transaction. mutable std::atomic_bool transactions_enabled = false; + std::atomic_bool data_parts_loading_finished = false; + /// Work with data parts struct TagByInfo{}; @@ -1242,6 +1245,9 @@ protected: /// Attaches restored parts to the storage. virtual void attachRestoredParts(MutableDataPartsVector && parts) = 0; + void resetObjectColumnsFromActiveParts(const DataPartsLock & lock); + void updateObjectColumns(const DataPartPtr & part, const DataPartsLock & lock); + static void incrementInsertedPartsProfileEvent(MergeTreeDataPartType type); static void incrementMergedPartsProfileEvent(MergeTreeDataPartType type); @@ -1329,9 +1335,6 @@ private: DataPartsVector & duplicate_parts_to_remove, MutableDataPartsVector & parts_from_wal); - void resetObjectColumnsFromActiveParts(const DataPartsLock & lock); - void updateObjectColumns(const DataPartPtr & part, const DataPartsLock & lock); - /// Create zero-copy exclusive lock for part and disk. Useful for coordination of /// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree. virtual std::optional tryCreateZeroCopyExclusiveLock(const String &, const DiskPtr &) { return std::nullopt; } diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 14b3e33a157..4a21ae1592a 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include @@ -47,9 +48,11 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( const ReadBufferFromFileBase::ProfileCallback & profile_callback) const { auto read_info = std::make_shared(shared_from_this()); + auto * load_marks_threadpool = reader_settings.read_settings.load_marks_asynchronously ? &read_info->getContext()->getLoadMarksThreadpool() : nullptr; + return std::make_unique( read_info, columns_to_read, metadata_snapshot, uncompressed_cache, - mark_cache, mark_ranges, reader_settings, + mark_cache, mark_ranges, reader_settings, load_marks_threadpool, avg_value_size_hints, profile_callback); } @@ -93,7 +96,7 @@ void MergeTreeDataPartCompact::calculateEachColumnSizes(ColumnSizeByName & /*eac void MergeTreeDataPartCompact::loadIndexGranularityImpl( MergeTreeIndexGranularity & index_granularity_, const MergeTreeIndexGranularityInfo & index_granularity_info_, - const NamesAndTypesList & columns_, const DataPartStoragePtr & data_part_storage_) + size_t columns_count, const DataPartStoragePtr & data_part_storage_) { if (!index_granularity_info_.is_adaptive) throw Exception("MergeTreeDataPartCompact cannot be created with non-adaptive granulary.", ErrorCodes::NOT_IMPLEMENTED); @@ -111,13 +114,13 @@ void MergeTreeDataPartCompact::loadIndexGranularityImpl( while (!buffer->eof()) { /// Skip offsets for columns - buffer->seek(columns_.size() * sizeof(MarkInCompressedFile), SEEK_CUR); + buffer->seek(columns_count * sizeof(MarkInCompressedFile), SEEK_CUR); size_t granularity; readIntBinary(granularity, *buffer); index_granularity_.appendMark(granularity); } - if (index_granularity_.getMarksCount() * index_granularity_info_.getMarkSizeInBytes(columns_.size()) != marks_file_size) + if (index_granularity_.getMarksCount() * index_granularity_info_.getMarkSizeInBytes(columns_count) != marks_file_size) throw Exception("Cannot read all marks from file " + marks_file_path, ErrorCodes::CANNOT_READ_ALL_DATA); index_granularity_.setInitialized(); @@ -128,7 +131,7 @@ void MergeTreeDataPartCompact::loadIndexGranularity() if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - loadIndexGranularityImpl(index_granularity, index_granularity_info, columns, data_part_storage); + loadIndexGranularityImpl(index_granularity, index_granularity_info, columns.size(), data_part_storage); } bool MergeTreeDataPartCompact::hasColumnFiles(const NameAndTypePair & column) const diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 26c335f4324..d3ac71cb02a 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -68,7 +68,7 @@ public: protected: static void loadIndexGranularityImpl( MergeTreeIndexGranularity & index_granularity_, const MergeTreeIndexGranularityInfo & index_granularity_info_, - const NamesAndTypesList & columns_, const DataPartStoragePtr & data_part_storage_); + size_t columns_count, const DataPartStoragePtr & data_part_storage_); private: void checkConsistency(bool require_part_metadata) const override; diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 97900eef22b..95faef6aac7 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -483,16 +483,6 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( return temp_part; } -void MergeTreeDataWriter::deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block) -{ - if (!storage_snapshot->object_columns.empty()) - { - auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects(); - auto storage_columns = storage_snapshot->getColumns(options); - convertObjectsToTuples(block, storage_columns); - } -} - MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( const String & part_name, MergeTreeDataPartType part_type, diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h index 2f9ab1aae8b..00438a29fa1 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.h +++ b/src/Storages/MergeTree/MergeTreeDataWriter.h @@ -45,8 +45,6 @@ public: */ static BlocksWithPartition splitBlockIntoParts(const Block & block, size_t max_parts, const StorageMetadataPtr & metadata_snapshot, ContextPtr context); - static void deduceTypesOfObjectColumns(const StorageSnapshotPtr & storage_snapshot, Block & block); - /// This structure contains not completely written temporary part. /// Some writes may happen asynchronously, e.g. for blob storages. /// You should call finalize() to wait until all data is written. diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index 9d1a075a63f..dc890910224 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -242,11 +242,21 @@ bool MergeTreeIndexConditionBloomFilter::traverseAtomAST(const ASTPtr & node, Bl DataTypePtr const_type; if (KeyCondition::getConstant(node, block_with_constants, const_value, const_type)) { - if (const_value.getType() == Field::Types::UInt64 || const_value.getType() == Field::Types::Int64 || - const_value.getType() == Field::Types::Float64) + if (const_value.getType() == Field::Types::UInt64) { - /// Zero in all types is represented in memory the same way as in UInt64. - out.function = const_value.reinterpret() ? RPNElement::ALWAYS_TRUE : RPNElement::ALWAYS_FALSE; + out.function = const_value.get() ? RPNElement::ALWAYS_TRUE : RPNElement::ALWAYS_FALSE; + return true; + } + + if (const_value.getType() == Field::Types::Int64) + { + out.function = const_value.get() ? RPNElement::ALWAYS_TRUE : RPNElement::ALWAYS_FALSE; + return true; + } + + if (const_value.getType() == Field::Types::Float64) + { + out.function = const_value.get() ? RPNElement::ALWAYS_TRUE : RPNElement::ALWAYS_FALSE; return true; } } diff --git a/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/src/Storages/MergeTree/MergeTreeIndexReader.cpp index c43c75035e4..33106f7ab64 100644 --- a/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -1,4 +1,5 @@ #include +#include namespace { @@ -15,6 +16,9 @@ std::unique_ptr makeIndexReader( UncompressedCache * uncompressed_cache, MergeTreeReaderSettings settings) { + auto context = part->storage.getContext(); + auto * load_marks_threadpool = settings.read_settings.load_marks_asynchronously ? &context->getLoadMarksThreadpool() : nullptr; + return std::make_unique( part->data_part_storage, index->getFileName(), extension, marks_count, @@ -22,7 +26,7 @@ std::unique_ptr makeIndexReader( std::move(settings), mark_cache, uncompressed_cache, part->getFileSizeOrZero(index->getFileName() + extension), &part->index_granularity_info, - ReadBufferFromFileBase::ProfileCallback{}, CLOCK_MONOTONIC_COARSE, false); + ReadBufferFromFileBase::ProfileCallback{}, CLOCK_MONOTONIC_COARSE, false, load_marks_threadpool); } } diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index ce26a86f0c0..ad5d828c431 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -2,9 +2,19 @@ #include #include #include +#include +#include +#include +#include #include +namespace ProfileEvents +{ + extern const Event WaitMarksLoadMicroseconds; + extern const Event BackgroundLoadingMarksTasks; +} + namespace DB { @@ -23,6 +33,7 @@ MergeTreeMarksLoader::MergeTreeMarksLoader( const MergeTreeIndexGranularityInfo & index_granularity_info_, bool save_marks_in_cache_, const ReadSettings & read_settings_, + ThreadPool * load_marks_threadpool_, size_t columns_in_mark_) : data_part_storage(std::move(data_part_storage_)) , mark_cache(mark_cache_) @@ -32,13 +43,41 @@ MergeTreeMarksLoader::MergeTreeMarksLoader( , save_marks_in_cache(save_marks_in_cache_) , columns_in_mark(columns_in_mark_) , read_settings(read_settings_) + , load_marks_threadpool(load_marks_threadpool_) { + if (load_marks_threadpool) + { + future = loadMarksAsync(); + } +} + +MergeTreeMarksLoader::~MergeTreeMarksLoader() +{ + if (future.valid()) + { + future.wait(); + } } const MarkInCompressedFile & MergeTreeMarksLoader::getMark(size_t row_index, size_t column_index) { if (!marks) - loadMarks(); + { + Stopwatch watch(CLOCK_MONOTONIC); + + if (future.valid()) + { + marks = future.get(); + future = {}; + } + else + { + marks = loadMarks(); + } + + watch.stop(); + ProfileEvents::increment(ProfileEvents::WaitMarksLoadMicroseconds, watch.elapsedMicroseconds()); + } #ifndef NDEBUG if (column_index >= columns_in_mark) @@ -95,28 +134,63 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() return res; } -void MergeTreeMarksLoader::loadMarks() +MarkCache::MappedPtr MergeTreeMarksLoader::loadMarks() { + MarkCache::MappedPtr loaded_marks; + if (mark_cache) { auto key = mark_cache->hash(fs::path(data_part_storage->getFullPath()) / mrk_path); if (save_marks_in_cache) { auto callback = [this]{ return loadMarksImpl(); }; - marks = mark_cache->getOrSet(key, callback); + loaded_marks = mark_cache->getOrSet(key, callback); } else { - marks = mark_cache->get(key); - if (!marks) - marks = loadMarksImpl(); + loaded_marks = mark_cache->get(key); + if (!loaded_marks) + loaded_marks = loadMarksImpl(); } } else - marks = loadMarksImpl(); + loaded_marks = loadMarksImpl(); - if (!marks) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to load marks: {}", String(fs::path(data_part_storage->getFullPath()) / mrk_path)); + if (!loaded_marks) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Failed to load marks: {}", + (fs::path(data_part_storage->getFullPath()) / mrk_path).string()); + } + + return loaded_marks; +} + +std::future MergeTreeMarksLoader::loadMarksAsync() +{ + ThreadGroupStatusPtr thread_group; + if (CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup()) + thread_group = CurrentThread::get().getThreadGroup(); + + auto task = std::make_shared>([thread_group, this] + { + setThreadName("loadMarksThread"); + + if (thread_group) + CurrentThread::attachTo(thread_group); + + SCOPE_EXIT_SAFE({ + if (thread_group) + CurrentThread::detachQuery(); + }); + + ProfileEvents::increment(ProfileEvents::BackgroundLoadingMarksTasks); + return loadMarks(); + }); + + auto task_future = task->get_future(); + load_marks_threadpool->scheduleOrThrow([task]{ (*task)(); }); + return task_future; } } diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.h b/src/Storages/MergeTree/MergeTreeMarksLoader.h index 3a1d3dc2c1b..60ccc953e9b 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -2,11 +2,13 @@ #include #include #include +#include namespace DB { struct MergeTreeIndexGranularityInfo; +class Threadpool; class MergeTreeMarksLoader { @@ -21,8 +23,11 @@ public: const MergeTreeIndexGranularityInfo & index_granularity_info_, bool save_marks_in_cache_, const ReadSettings & read_settings_, + ThreadPool * load_marks_threadpool_, size_t columns_in_mark_ = 1); + ~MergeTreeMarksLoader(); + const MarkInCompressedFile & getMark(size_t row_index, size_t column_index = 0); private: @@ -36,8 +41,12 @@ private: MarkCache::MappedPtr marks; ReadSettings read_settings; - void loadMarks(); + MarkCache::MappedPtr loadMarks(); + std::future loadMarksAsync(); MarkCache::MappedPtr loadMarksImpl(); + + std::future future; + ThreadPool * load_marks_threadpool; }; } diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index a723fa6d8e3..860e28e4da4 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -211,12 +211,11 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt DataPartStoragePtr cloned_part_storage; - const String directory_to_move = "moving"; if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication) { /// Try zero-copy replication and fallback to default copy if it's not possible moving_part.part->assertOnDisk(); - String path_to_clone = fs::path(data->getRelativeDataPath()) / directory_to_move / ""; + String path_to_clone = fs::path(data->getRelativeDataPath()) / MergeTreeData::MOVING_DIR_NAME / ""; String relative_path = part->data_part_storage->getPartDirectory(); if (disk->exists(path_to_clone + relative_path)) { @@ -236,7 +235,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt } else { - cloned_part_storage = part->makeCloneOnDisk(disk, directory_to_move); + cloned_part_storage = part->makeCloneOnDisk(disk, MergeTreeData::MOVING_DIR_NAME); } MergeTreeData::MutableDataPartPtr cloned_part = data->createPart(part->name, cloned_part_storage); diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index d59da08aa6c..4801c9a4058 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -22,6 +22,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( MarkCache * mark_cache_, MarkRanges mark_ranges_, MergeTreeReaderSettings settings_, + ThreadPool * load_marks_threadpool_, ValueSizeMap avg_value_size_hints_, const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) @@ -42,6 +43,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( data_part_info_for_read_->getIndexGranularityInfo(), settings.save_marks_in_cache, settings.read_settings, + load_marks_threadpool_, data_part_info_for_read_->getColumns().size()) { try diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 953455b7e26..ee099755a8e 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -26,6 +26,7 @@ public: MarkCache * mark_cache_, MarkRanges mark_ranges_, MergeTreeReaderSettings settings_, + ThreadPool * load_marks_threadpool_, ValueSizeMap avg_value_size_hints_ = {}, const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = {}, clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE); diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index d9913f2639f..47f8b0f6008 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -16,14 +16,19 @@ namespace ErrorCodes MergeTreeReaderStream::MergeTreeReaderStream( DataPartStoragePtr data_part_storage_, - const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, + const String & path_prefix_, + const String & data_file_extension_, + size_t marks_count_, const MarkRanges & all_mark_ranges_, const MergeTreeReaderSettings & settings_, MarkCache * mark_cache_, - UncompressedCache * uncompressed_cache_, size_t file_size_, + UncompressedCache * uncompressed_cache_, + size_t file_size_, const MergeTreeIndexGranularityInfo * index_granularity_info_, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_, - bool is_low_cardinality_dictionary_) + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, + clockid_t clock_type_, + bool is_low_cardinality_dictionary_, + ThreadPool * load_marks_cache_threadpool_) : settings(settings_) , profile_callback(profile_callback_) , clock_type(clock_type_) @@ -45,7 +50,8 @@ MergeTreeReaderStream::MergeTreeReaderStream( marks_count, *index_granularity_info, save_marks_in_cache, - settings.read_settings) + settings.read_settings, + load_marks_cache_threadpool_) { } diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index f5a8ebadcba..83e314eef42 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -20,13 +20,19 @@ class MergeTreeReaderStream public: MergeTreeReaderStream( DataPartStoragePtr data_part_storage_, - const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, + const String & path_prefix_, + const String & data_file_extension_, + size_t marks_count_, const MarkRanges & all_mark_ranges, const MergeTreeReaderSettings & settings_, - MarkCache * mark_cache, UncompressedCache * uncompressed_cache, - size_t file_size_, const MergeTreeIndexGranularityInfo * index_granularity_info_, - const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type, - bool is_low_cardinality_dictionary_); + MarkCache * mark_cache, + UncompressedCache * uncompressed_cache, + size_t file_size_, + const MergeTreeIndexGranularityInfo * index_granularity_info_, + const ReadBufferFromFileBase::ProfileCallback & profile_callback, + clockid_t clock_type, + bool is_low_cardinality_dictionary_, + ThreadPool * load_marks_cache_threadpool_); void seekToMark(size_t index); diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 22f07e26473..ea367a9502e 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -186,12 +187,15 @@ void MergeTreeReaderWide::addStreams( has_any_stream = true; bool is_lc_dict = substream_path.size() > 1 && substream_path[substream_path.size() - 2].type == ISerialization::Substream::Type::DictionaryKeys; + auto context = data_part_info_for_read->getContext(); + auto * load_marks_threadpool = settings.read_settings.load_marks_asynchronously ? &context->getLoadMarksThreadpool() : nullptr; + streams.emplace(stream_name, std::make_unique( data_part_info_for_read->getDataPartStorage(), stream_name, DATA_FILE_EXTENSION, data_part_info_for_read->getMarksCount(), all_mark_ranges, settings, mark_cache, uncompressed_cache, data_part_info_for_read->getFileSizeOrZero(stream_name + DATA_FILE_EXTENSION), &data_part_info_for_read->getIndexGranularityInfo(), - profile_callback, clock_type, is_lc_dict)); + profile_callback, clock_type, is_lc_dict, load_marks_threadpool)); }; serialization->enumerateStreams(callback); diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 5eaa8ec8004..5d00db861a8 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include namespace ProfileEvents @@ -56,7 +57,7 @@ void MergeTreeSink::consume(Chunk chunk) { auto block = getHeader().cloneWithColumns(chunk.detachColumns()); - storage.writer.deduceTypesOfObjectColumns(storage_snapshot, block); + deduceTypesOfObjectColumns(storage_snapshot, block); auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); using DelayedPartitions = std::vector; diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index c8b3349734e..4735eae8fdd 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -59,6 +59,18 @@ MergeTreeWriteAheadLog::~MergeTreeWriteAheadLog() } } + +void MergeTreeWriteAheadLog::dropAllWriteAheadLogs(DiskPtr disk_to_drop, std::string relative_data_path) +{ + std::vector files; + disk_to_drop->listFiles(relative_data_path, files); + for (const auto & file : files) + { + if (file.starts_with(WAL_FILE_NAME)) + disk_to_drop->removeFile(fs::path(relative_data_path) / file); + } +} + void MergeTreeWriteAheadLog::init() { out = disk->writeFile(path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Append); diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.h b/src/Storages/MergeTree/MergeTreeWriteAheadLog.h index b54161dbdaa..a03fe09e03d 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.h +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.h @@ -71,6 +71,8 @@ public: static std::optional tryParseMinMaxBlockNumber(const String & filename); void shutdown(); + /// Drop all write ahead logs from disk. Useful during table drop. + static void dropAllWriteAheadLogs(DiskPtr disk_to_drop, std::string relative_data_path); private: void init(); void rotate(const std::unique_lock & lock); diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index a51eb7854ab..fc8b22865c4 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -134,6 +134,29 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() .part_log_writer = {} }; } + else if (!storage.findReplicaHavingCoveringPart(entry.new_part_name, /* active */ false, dummy).empty()) + { + /// Why this if still needed? We can check for part in zookeeper, don't find it and sleep for any amount of time. During this sleep part will be actually committed from other replica + /// and exclusive zero copy lock will be released. We will take the lock and execute mutation one more time, while it was possible just to download the part from other replica. + /// + /// It's also possible just because reads in [Zoo]Keeper are not lineariazable. + /// + /// NOTE: In case of mutation and hardlinks it can even lead to extremely rare dataloss (we will produce new part with the same hardlinks, don't fetch the same from other replica), so this check is important. + /// + /// In case of DROP_RANGE on fast replica and stale replica we can have some failed select queries in case of zero copy replication. + zero_copy_lock->lock->unlock(); + + LOG_DEBUG(log, "We took zero copy lock, but mutation of part {} finished by some other replica, will release lock and download mutated part to avoid data duplication", entry.new_part_name); + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; + } + else + { + LOG_DEBUG(log, "Zero copy lock taken, will mutate part {}", entry.new_part_name); + } } } diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.h b/src/Storages/MergeTree/MutateFromLogEntryTask.h index a0bbaabda85..416b0c92522 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.h +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -18,7 +19,12 @@ public: ReplicatedMergeTreeQueue::SelectedEntryPtr selected_entry_, StorageReplicatedMergeTree & storage_, Callback && task_result_callback_) - : ReplicatedMergeMutateTaskBase(&Poco::Logger::get("MutateFromLogEntryTask"), storage_, selected_entry_, task_result_callback_) {} + : ReplicatedMergeMutateTaskBase( + &Poco::Logger::get(storage_.getStorageID().getShortName() + "::" + selected_entry_->log_entry->new_part_name + "(MutateFromLogEntryTask)"), + storage_, + selected_entry_, + task_result_callback_) + {} UInt64 getPriority() override { return priority; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 6c7fbcb52d8..b9bd027cde2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -161,7 +162,7 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) */ size_t replicas_num = checkQuorumPrecondition(zookeeper); - storage.writer.deduceTypesOfObjectColumns(storage_snapshot, block); + deduceTypesOfObjectColumns(storage_snapshot, block); auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); using DelayedPartitions = std::vector; @@ -203,11 +204,11 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) } block_id = temp_part.part->getZeroLevelPartBlockID(block_dedup_token); - LOG_DEBUG(log, "Wrote block with ID '{}', {} rows on {} replicas", block_id, current_block.block.rows(), replicas_num); + LOG_DEBUG(log, "Wrote block with ID '{}', {} rows{}", block_id, current_block.block.rows(), quorumLogMessage(replicas_num)); } else { - LOG_DEBUG(log, "Wrote block with {} rows on {} replicas", current_block.block.rows(), replicas_num); + LOG_DEBUG(log, "Wrote block with {} rows{}", current_block.block.rows(), quorumLogMessage(replicas_num)); } UInt64 elapsed_ns = watch.elapsed(); @@ -639,7 +640,7 @@ void ReplicatedMergeTreeSink::waitForQuorum( size_t replicas_num) const { /// We are waiting for quorum to be satisfied. - LOG_TRACE(log, "Waiting for quorum '{}' for part {} on {} replicas", quorum_path, part_name, replicas_num); + LOG_TRACE(log, "Waiting for quorum '{}' for part {}{}", quorum_path, part_name, quorumLogMessage(replicas_num)); try { @@ -684,6 +685,13 @@ void ReplicatedMergeTreeSink::waitForQuorum( LOG_TRACE(log, "Quorum '{}' for part {} satisfied", quorum_path, part_name); } +String ReplicatedMergeTreeSink::quorumLogMessage(size_t replicas_num) const +{ + if (!isQuorumEnabled()) + return ""; + return fmt::format(" (quorum {} of {} replicas)", getQuorumSize(replicas_num), replicas_num); +} + size_t ReplicatedMergeTreeSink::getQuorumSize(size_t replicas_num) const { if (!isQuorumEnabled()) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h index 48e94ef5659..ab729e6edec 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h @@ -96,6 +96,7 @@ private: size_t getQuorumSize(size_t replicas_num) const; bool isQuorumEnabled() const; + String quorumLogMessage(size_t replicas_num) const; /// Used in logs for debug purposes size_t quorum_timeout_ms; size_t max_parts_per_block; diff --git a/src/Storages/NATS/NATSSettings.h b/src/Storages/NATS/NATSSettings.h index 6029aaea9f6..9bf9b969387 100644 --- a/src/Storages/NATS/NATSSettings.h +++ b/src/Storages/NATS/NATSSettings.h @@ -24,7 +24,8 @@ class ASTStorage; M(Milliseconds, nats_flush_interval_ms, 0, "Timeout for flushing data from NATS.", 0) \ M(String, nats_username, "", "NATS username", 0) \ M(String, nats_password, "", "NATS password", 0) \ - M(String, nats_token, "", "NATS token", 0) + M(String, nats_token, "", "NATS token", 0) \ + M(UInt64, nats_startup_connect_tries, 5, "Number of connect tries at startup", 0) \ #define LIST_OF_NATS_SETTINGS(M) \ NATS_RELATED_SETTINGS(M) \ diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp index fc3079a7aa7..4a3ba973e67 100644 --- a/src/Storages/NATS/StorageNATS.cpp +++ b/src/Storages/NATS/StorageNATS.cpp @@ -92,10 +92,24 @@ StorageNATS::StorageNATS( try { - connection = std::make_shared(configuration, log); - if (!connection->connect()) - throw Exception(ErrorCodes::CANNOT_CONNECT_NATS, "Cannot connect to {}. Nats last error: {}", - connection->connectionInfoForLog(), nats_GetLastError(nullptr)); + size_t num_tries = nats_settings->nats_startup_connect_tries; + for (size_t i = 0; i < num_tries; ++i) + { + connection = std::make_shared(configuration, log); + + if (connection->connect()) + break; + + if (i == num_tries - 1) + { + throw Exception( + ErrorCodes::CANNOT_CONNECT_NATS, + "Cannot connect to {}. Nats last error: {}", + connection->connectionInfoForLog(), nats_GetLastError(nullptr)); + } + + LOG_DEBUG(log, "Connect attempt #{} failed, error: {}. Reconnecting...", i + 1, nats_GetLastError(nullptr)); + } } catch (...) { diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index 31cb2f2f9c2..20b1de51a30 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -523,8 +523,7 @@ Chunk StorageEmbeddedRocksDB::getByKeys( Block StorageEmbeddedRocksDB::getSampleBlock(const Names &) const { - auto metadata = getInMemoryMetadataPtr(); - return metadata ? metadata->getSampleBlock() : Block(); + return getInMemoryMetadataPtr()->getSampleBlock(); } Chunk StorageEmbeddedRocksDB::getBySerializedKeys( diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index c41b422199d..f2835ab4dbf 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -117,10 +117,10 @@ struct InputOrderInfo * sort_description_for_merging will be equal to (c, d) and * used_prefix_of_sorting_key_size will be equal to 4. */ - size_t used_prefix_of_sorting_key_size; + const size_t used_prefix_of_sorting_key_size; - int direction; - UInt64 limit; + const int direction; + const UInt64 limit; InputOrderInfo( const SortDescription & sort_description_for_merging_, diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp new file mode 100644 index 00000000000..3ae7cf7a7e4 --- /dev/null +++ b/src/Storages/StorageKeeperMap.cpp @@ -0,0 +1,771 @@ +#include + +#include + +#include + +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; + extern const int KEEPER_EXCEPTION; + extern const int LOGICAL_ERROR; + extern const int LIMIT_EXCEEDED; +} + +namespace +{ + +std::string formattedAST(const ASTPtr & ast) +{ + if (!ast) + return ""; + return serializeAST(*ast); +} + +void verifyTableId(const StorageID & table_id) +{ + if (!table_id.hasUUID()) + { + auto database = DatabaseCatalog::instance().getDatabase(table_id.database_name); + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "KeeperMap cannot be used with '{}' database because it uses {} engine. Please use Atomic or Replicated database", + table_id.getDatabaseName(), + database->getEngineName()); + } + +} + +} + +class StorageKeeperMapSink : public SinkToStorage +{ + StorageKeeperMap & storage; + std::unordered_map new_values; + size_t primary_key_pos; + +public: + StorageKeeperMapSink(StorageKeeperMap & storage_, const StorageMetadataPtr & metadata_snapshot) + : SinkToStorage(metadata_snapshot->getSampleBlock()), storage(storage_) + { + auto primary_key = storage.getPrimaryKey(); + assert(primary_key.size() == 1); + primary_key_pos = getHeader().getPositionByName(primary_key[0]); + } + + std::string getName() const override { return "StorageKeeperMapSink"; } + + void consume(Chunk chunk) override + { + auto rows = chunk.getNumRows(); + auto block = getHeader().cloneWithColumns(chunk.detachColumns()); + + WriteBufferFromOwnString wb_key; + WriteBufferFromOwnString wb_value; + + for (size_t i = 0; i < rows; ++i) + { + wb_key.restart(); + wb_value.restart(); + + size_t idx = 0; + for (const auto & elem : block) + { + elem.type->getDefaultSerialization()->serializeBinary(*elem.column, i, idx == primary_key_pos ? wb_key : wb_value); + ++idx; + } + + auto key = base64Encode(wb_key.str(), /* url_encoding */ true); + new_values[std::move(key)] = std::move(wb_value.str()); + } + } + + void onFinish() override + { + auto zookeeper = storage.getClient(); + + Coordination::Requests requests; + + auto keys_limit = storage.keysLimit(); + + size_t current_keys_num = 0; + size_t new_keys_num = 0; + + // We use keys limit as a soft limit so we ignore some cases when it can be still exceeded + // (e.g if parallel insert queries are being run) + if (keys_limit != 0) + { + Coordination::Stat data_stat; + zookeeper->get(storage.dataPath(), &data_stat); + current_keys_num = data_stat.numChildren; + } + + std::vector>> exist_responses; + for (const auto & [key, value] : new_values) + { + auto path = storage.fullPathForKey(key); + + exist_responses.push_back({&key, zookeeper->asyncExists(path)}); + } + + for (auto & [key, response] : exist_responses) + { + if (response.get().error == Coordination::Error::ZOK) + { + requests.push_back(zkutil::makeSetRequest(storage.fullPathForKey(*key), new_values[*key], -1)); + } + else + { + requests.push_back(zkutil::makeCreateRequest(storage.fullPathForKey(*key), new_values[*key], zkutil::CreateMode::Persistent)); + ++new_keys_num; + } + } + + if (new_keys_num != 0) + { + auto will_be = current_keys_num + new_keys_num; + if (keys_limit != 0 && will_be > keys_limit) + throw Exception( + ErrorCodes::LIMIT_EXCEEDED, + "Limit would be exceeded by inserting {} new key(s). Limit is {}, while the number of keys would be {}", + new_keys_num, + keys_limit, + will_be); + } + + zookeeper->multi(requests); + } +}; + +template +class StorageKeeperMapSource : public ISource +{ + const StorageKeeperMap & storage; + size_t max_block_size; + + using KeyContainerPtr = std::shared_ptr; + KeyContainerPtr container; + using KeyContainerIter = typename KeyContainer::const_iterator; + KeyContainerIter it; + KeyContainerIter end; + +public: + StorageKeeperMapSource( + const StorageKeeperMap & storage_, + const Block & header, + size_t max_block_size_, + KeyContainerPtr container_, + KeyContainerIter begin_, + KeyContainerIter end_) + : ISource(header), storage(storage_), max_block_size(max_block_size_), container(std::move(container_)), it(begin_), end(end_) + { + } + + std::string getName() const override { return "StorageKeeperMapSource"; } + + Chunk generate() override + { + if (it >= end) + { + it = {}; + return {}; + } + + using KeyType = typename KeyContainer::value_type; + if constexpr (std::same_as) + { + const auto & sample_block = getPort().getHeader(); + const auto & key_column_type = sample_block.getByName(storage.getPrimaryKey().at(0)).type; + auto raw_keys = serializeKeysToRawString(it, end, key_column_type, max_block_size); + + for (auto & raw_key : raw_keys) + raw_key = base64Encode(raw_key, /* url_encoding */ true); + + return storage.getBySerializedKeys(raw_keys, nullptr); + } + else + { + size_t elem_num = std::min(max_block_size, static_cast(end - it)); + auto chunk = storage.getBySerializedKeys(std::span{it, it + elem_num}, nullptr); + it += elem_num; + return chunk; + } + } +}; + +StorageKeeperMap::StorageKeeperMap( + ContextPtr context_, + const StorageID & table_id, + const StorageInMemoryMetadata & metadata, + bool attach, + std::string_view primary_key_, + const std::string & root_path_, + UInt64 keys_limit_) + : IStorage(table_id) + , WithContext(context_->getGlobalContext()) + , root_path(zkutil::extractZooKeeperPath(root_path_, false)) + , primary_key(primary_key_) + , zookeeper_name(zkutil::extractZooKeeperName(root_path_)) + , keys_limit(keys_limit_) + , log(&Poco::Logger::get(fmt::format("StorageKeeperMap ({})", table_id.getNameForLogs()))) +{ + std::string path_prefix = context_->getConfigRef().getString("keeper_map_path_prefix", ""); + if (path_prefix.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "KeeperMap is disabled because 'keeper_map_path_prefix' config is not defined"); + + verifyTableId(table_id); + + setInMemoryMetadata(metadata); + + WriteBufferFromOwnString out; + out << "KeeperMap metadata format version: 1\n" + << "columns: " << metadata.columns.toString() + << "primary key: " << formattedAST(metadata.getPrimaryKey().expression_list_ast) << "\n"; + metadata_string = out.str(); + + if (root_path.empty()) + throw Exception("root_path should not be empty", ErrorCodes::BAD_ARGUMENTS); + if (!root_path.starts_with('/')) + throw Exception("root_path should start with '/'", ErrorCodes::BAD_ARGUMENTS); + + auto config_keys_limit = context_->getConfigRef().getUInt64("keeper_map_keys_limit", 0); + if (config_keys_limit != 0 && (keys_limit == 0 || keys_limit > config_keys_limit)) + { + LOG_WARNING( + log, + "Keys limit defined by argument ({}) is larger than the one defined by 'keeper_map_keys_limit' config ({}). Will use " + "config defined value", + keys_limit, + config_keys_limit); + keys_limit = config_keys_limit; + } + else if (keys_limit > 0) + { + LOG_INFO(log, "Keys limit will be set to {}", keys_limit); + } + + auto root_path_fs = fs::path(path_prefix) / std::string_view{root_path}.substr(1); + root_path = root_path_fs.generic_string(); + + data_path = root_path_fs / "data"; + + auto metadata_path_fs = root_path_fs / "metadata"; + metadata_path = metadata_path_fs; + tables_path = metadata_path_fs / "tables"; + + auto table_unique_id = toString(table_id.uuid) + toString(ServerUUID::get()); + table_path = fs::path(tables_path) / table_unique_id; + + dropped_path = metadata_path_fs / "dropped"; + dropped_lock_path = fs::path(dropped_path) / "lock"; + + if (attach) + { + checkTable(); + return; + } + + auto client = getClient(); + + if (root_path != "/" && !client->exists(root_path)) + { + LOG_TRACE(log, "Creating root path {}", root_path); + client->createAncestors(root_path); + client->createIfNotExists(root_path, ""); + } + + for (size_t i = 0; i < 1000; ++i) + { + if (client->exists(dropped_path)) + { + LOG_INFO(log, "Removing leftover nodes"); + auto code = client->tryCreate(dropped_lock_path, "", zkutil::CreateMode::Ephemeral); + + if (code == Coordination::Error::ZNONODE) + { + LOG_INFO(log, "Someone else removed leftover nodes"); + } + else if (code == Coordination::Error::ZNODEEXISTS) + { + LOG_INFO(log, "Someone else is removing leftover nodes"); + continue; + } + else if (code != Coordination::Error::ZOK) + { + throw Coordination::Exception(code, dropped_lock_path); + } + else + { + auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(dropped_lock_path, *client); + if (!dropTable(client, metadata_drop_lock)) + continue; + } + } + + std::string stored_metadata_string; + auto exists = client->tryGet(metadata_path, stored_metadata_string); + + if (exists) + { + // this requires same name for columns + // maybe we can do a smarter comparison for columns and primary key expression + if (stored_metadata_string != metadata_string) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Path {} is already used but the stored table definition doesn't match. Stored metadata: {}", + root_path, + stored_metadata_string); + } + else + { + auto code = client->tryCreate(metadata_path, metadata_string, zkutil::CreateMode::Persistent); + if (code == Coordination::Error::ZNODEEXISTS) + continue; + else if (code != Coordination::Error::ZOK) + throw Coordination::Exception(code, metadata_path); + } + + client->createIfNotExists(tables_path, ""); + + auto code = client->tryCreate(table_path, "", zkutil::CreateMode::Persistent); + + if (code == Coordination::Error::ZOK) + { + // metadata now should be guaranteed to exist because we added our UUID to the tables_path + client->createIfNotExists(data_path, ""); + table_is_valid = true; + return; + } + + if (code == Coordination::Error::ZNONODE) + LOG_INFO(log, "Metadata nodes were deleted in background, will retry"); + else + throw Coordination::Exception(code, table_path); + } + + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot create metadata for table, because it is removed concurrently or because of wrong root_path ({})", root_path); +} + + +Pipe StorageKeeperMap::read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context_, + QueryProcessingStage::Enum /*processed_stage*/, + size_t max_block_size, + unsigned num_streams) +{ + checkTable(); + storage_snapshot->check(column_names); + + FieldVectorPtr filtered_keys; + bool all_scan; + + Block sample_block = storage_snapshot->metadata->getSampleBlock(); + auto primary_key_type = sample_block.getByName(primary_key).type; + std::tie(filtered_keys, all_scan) = getFilterKeys(primary_key, primary_key_type, query_info, context_); + + const auto process_keys = [&](KeyContainerPtr keys) -> Pipe + { + if (keys->empty()) + return {}; + + ::sort(keys->begin(), keys->end()); + keys->erase(std::unique(keys->begin(), keys->end()), keys->end()); + + Pipes pipes; + + size_t num_keys = keys->size(); + size_t num_threads = std::min(num_streams, keys->size()); + + assert(num_keys <= std::numeric_limits::max()); + assert(num_threads <= std::numeric_limits::max()); + + for (size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) + { + size_t begin = num_keys * thread_idx / num_threads; + size_t end = num_keys * (thread_idx + 1) / num_threads; + + using KeyContainer = typename KeyContainerPtr::element_type; + pipes.emplace_back(std::make_shared>( + *this, sample_block, max_block_size, keys, keys->begin() + begin, keys->begin() + end)); + } + return Pipe::unitePipes(std::move(pipes)); + }; + + auto client = getClient(); + if (all_scan) + return process_keys(std::make_shared>(client->getChildren(data_path))); + + return process_keys(std::move(filtered_keys)); +} + +SinkToStoragePtr StorageKeeperMap::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr /*context*/) +{ + checkTable(); + return std::make_shared(*this, metadata_snapshot); +} + +void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) +{ + checkTable(); + auto client = getClient(); + client->tryRemoveChildrenRecursive(data_path, true); +} + +bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::EphemeralNodeHolder::Ptr & metadata_drop_lock) +{ + zookeeper->removeChildrenRecursive(data_path); + + bool completely_removed = false; + Coordination::Requests ops; + ops.emplace_back(zkutil::makeRemoveRequest(metadata_drop_lock->getPath(), -1)); + ops.emplace_back(zkutil::makeRemoveRequest(dropped_path, -1)); + ops.emplace_back(zkutil::makeRemoveRequest(data_path, -1)); + ops.emplace_back(zkutil::makeRemoveRequest(metadata_path, -1)); + + Coordination::Responses responses; + auto code = zookeeper->tryMulti(ops, responses); + using enum Coordination::Error; + switch (code) + { + case ZOK: + { + metadata_drop_lock->setAlreadyRemoved(); + completely_removed = true; + LOG_INFO(log, "Metadata ({}) and data ({}) was successfully removed from ZooKeeper", metadata_path, data_path); + break; + } + case ZNONODE: + throw Exception(ErrorCodes::LOGICAL_ERROR, "There is a race condition between creation and removal of metadata. It's a bug"); + case ZNOTEMPTY: + LOG_ERROR(log, "Metadata was not completely removed from ZooKeeper"); + break; + default: + zkutil::KeeperMultiException::check(code, ops, responses); + break; + } + return completely_removed; +} + +void StorageKeeperMap::drop() +{ + checkTable(); + auto client = getClient(); + + client->remove(table_path); + + if (!client->getChildren(tables_path).empty()) + return; + + Coordination::Requests ops; + Coordination::Responses responses; + + ops.emplace_back(zkutil::makeRemoveRequest(tables_path, -1)); + ops.emplace_back(zkutil::makeCreateRequest(dropped_path, "", zkutil::CreateMode::Persistent)); + ops.emplace_back(zkutil::makeCreateRequest(dropped_lock_path, "", zkutil::CreateMode::Ephemeral)); + + auto code = client->tryMulti(ops, responses); + + if (code == Coordination::Error::ZNONODE || code == Coordination::Error::ZNODEEXISTS) + { + LOG_INFO(log, "Metadata is being removed by another table"); + return; + } + else if (code == Coordination::Error::ZNOTEMPTY) + { + LOG_WARNING(log, "Another table is using the same path, metadata will not be deleted"); + return; + } + else if (code != Coordination::Error::ZOK) + zkutil::KeeperMultiException::check(code, ops, responses); + + auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(dropped_lock_path, *client); + dropTable(client, metadata_drop_lock); +} + +zkutil::ZooKeeperPtr StorageKeeperMap::getClient() const +{ + std::lock_guard lock{zookeeper_mutex}; + if (!zookeeper_client || zookeeper_client->expired()) + { + zookeeper_client = nullptr; + if (zookeeper_name == "default") + zookeeper_client = getContext()->getZooKeeper(); + else + zookeeper_client = getContext()->getAuxiliaryZooKeeper(zookeeper_name); + + zookeeper_client->sync(root_path); + } + + return zookeeper_client; +} + +const std::string & StorageKeeperMap::dataPath() const +{ + return data_path; +} + +std::string StorageKeeperMap::fullPathForKey(const std::string_view key) const +{ + return fs::path(data_path) / key; +} + +UInt64 StorageKeeperMap::keysLimit() const +{ + return keys_limit; +} + +std::optional StorageKeeperMap::isTableValid() const +{ + std::lock_guard lock{init_mutex}; + if (table_is_valid.has_value()) + return *table_is_valid; + + [&] + { + try + { + auto client = getClient(); + + std::string stored_metadata_string; + Coordination::Stat metadata_stat; + client->tryGet(metadata_path, stored_metadata_string, &metadata_stat); + + if (metadata_stat.numChildren == 0) + { + table_is_valid = false; + return; + } + + if (metadata_string != stored_metadata_string) + { + LOG_ERROR( + log, + "Table definition does not match to the one stored in the path {}. Stored definition: {}", + root_path, + stored_metadata_string); + table_is_valid = false; + return; + } + + // validate all metadata and data nodes are present + Coordination::Requests requests; + requests.push_back(zkutil::makeCheckRequest(table_path, -1)); + requests.push_back(zkutil::makeCheckRequest(data_path, -1)); + requests.push_back(zkutil::makeCheckRequest(dropped_path, -1)); + + Coordination::Responses responses; + client->tryMulti(requests, responses); + + table_is_valid = false; + if (responses[0]->error != Coordination::Error::ZOK) + { + LOG_ERROR(log, "Table node ({}) is missing", table_path); + return; + } + + if (responses[1]->error != Coordination::Error::ZOK) + { + LOG_ERROR(log, "Data node ({}) is missing", data_path); + return; + } + + if (responses[2]->error == Coordination::Error::ZOK) + { + LOG_ERROR(log, "Tables with root node {} are being dropped", root_path); + return; + } + + table_is_valid = true; + } + catch (const Coordination::Exception & e) + { + tryLogCurrentException(log); + + if (!Coordination::isHardwareError(e.code)) + table_is_valid = false; + } + }(); + + return table_is_valid; +} + +Chunk StorageKeeperMap::getByKeys(const ColumnsWithTypeAndName & keys, PaddedPODArray & null_map, const Names &) const +{ + if (keys.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "StorageKeeperMap supports only one key, got: {}", keys.size()); + + auto raw_keys = serializeKeysToRawString(keys[0]); + + if (raw_keys.size() != keys[0].column->size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Assertion failed: {} != {}", raw_keys.size(), keys[0].column->size()); + + return getBySerializedKeys(raw_keys, &null_map); +} + +Chunk StorageKeeperMap::getBySerializedKeys(const std::span keys, PaddedPODArray * null_map) const +{ + Block sample_block = getInMemoryMetadataPtr()->getSampleBlock(); + MutableColumns columns = sample_block.cloneEmptyColumns(); + size_t primary_key_pos = getPrimaryKeyPos(sample_block, getPrimaryKey()); + + if (null_map) + { + null_map->clear(); + null_map->resize_fill(keys.size(), 1); + } + + auto client = getClient(); + + std::vector> values; + values.reserve(keys.size()); + + for (const auto & key : keys) + { + const auto full_path = fullPathForKey(key); + values.emplace_back(client->asyncTryGet(full_path)); + } + + auto wait_until = std::chrono::system_clock::now() + std::chrono::milliseconds(Coordination::DEFAULT_OPERATION_TIMEOUT_MS); + + for (size_t i = 0; i < keys.size(); ++i) + { + auto & value = values[i]; + if (value.wait_until(wait_until) != std::future_status::ready) + throw DB::Exception(ErrorCodes::KEEPER_EXCEPTION, "Failed to fetch values: timeout"); + + auto response = value.get(); + Coordination::Error code = response.error; + + if (code == Coordination::Error::ZOK) + { + fillColumns(base64Decode(keys[i], true), response.data, primary_key_pos, sample_block, columns); + } + else if (code == Coordination::Error::ZNONODE) + { + if (null_map) + { + (*null_map)[i] = 0; + for (size_t col_idx = 0; col_idx < sample_block.columns(); ++col_idx) + columns[col_idx]->insert(sample_block.getByPosition(col_idx).type->getDefault()); + } + } + else + { + throw DB::Exception(ErrorCodes::KEEPER_EXCEPTION, "Failed to fetch value: {}", code); + } + } + + size_t num_rows = columns.at(0)->size(); + return Chunk(std::move(columns), num_rows); +} + +Block StorageKeeperMap::getSampleBlock(const Names &) const +{ + auto metadata = getInMemoryMetadataPtr(); + return metadata->getSampleBlock(); +} + +void StorageKeeperMap::checkTableCanBeRenamed(const StorageID & new_name) const +{ + verifyTableId(new_name); +} + +void StorageKeeperMap::rename(const String & /*new_path_to_table_data*/, const StorageID & new_table_id) +{ + checkTableCanBeRenamed(new_table_id); + renameInMemory(new_table_id); +} + +namespace +{ + +StoragePtr create(const StorageFactory::Arguments & args) +{ + ASTs & engine_args = args.engine_args; + if (engine_args.empty() || engine_args.size() > 2) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Storage KeeperMap requires 1-3 arguments:\n" + "root_path: path in the Keeper where the values will be stored (required)\n" + "keys_limit: number of keys allowed to be stored, 0 is no limit (default: 0)"); + + const auto root_path_node = evaluateConstantExpressionAsLiteral(engine_args[0], args.getLocalContext()); + auto root_path = checkAndGetLiteralArgument(root_path_node, "root_path"); + + UInt64 keys_limit = 0; + if (engine_args.size() > 1) + keys_limit = checkAndGetLiteralArgument(engine_args[1], "keys_limit"); + + StorageInMemoryMetadata metadata; + metadata.setColumns(args.columns); + metadata.setConstraints(args.constraints); + + if (!args.storage_def->primary_key) + throw Exception("StorageKeeperMap requires one column in primary key", ErrorCodes::BAD_ARGUMENTS); + + metadata.primary_key = KeyDescription::getKeyFromAST(args.storage_def->primary_key->ptr(), metadata.columns, args.getContext()); + auto primary_key_names = metadata.getColumnsRequiredForPrimaryKey(); + if (primary_key_names.size() != 1) + throw Exception("StorageKeeperMap requires one column in primary key", ErrorCodes::BAD_ARGUMENTS); + + return std::make_shared( + args.getContext(), args.table_id, metadata, args.query.attach, primary_key_names[0], root_path, keys_limit); +} + +} + +void registerStorageKeeperMap(StorageFactory & factory) +{ + factory.registerStorage( + "KeeperMap", + create, + { + .supports_sort_order = true, + .supports_parallel_insert = true, + }); +} + +} diff --git a/src/Storages/StorageKeeperMap.h b/src/Storages/StorageKeeperMap.h new file mode 100644 index 00000000000..87861362e42 --- /dev/null +++ b/src/Storages/StorageKeeperMap.h @@ -0,0 +1,138 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INVALID_STATE; +} + +// KV store using (Zoo|CH)Keeper +class StorageKeeperMap final : public IStorage, public IKeyValueEntity, WithContext +{ +public: + StorageKeeperMap( + ContextPtr context_, + const StorageID & table_id, + const StorageInMemoryMetadata & metadata, + bool attach, + std::string_view primary_key_, + const std::string & root_path_, + UInt64 keys_limit_); + + Pipe read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + + SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) override; + + void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) override; + void drop() override; + + std::string getName() const override { return "KeeperMap"; } + Names getPrimaryKey() const override { return {primary_key}; } + + Chunk getByKeys(const ColumnsWithTypeAndName & keys, PaddedPODArray & null_map, const Names &) const override; + Chunk getBySerializedKeys(std::span keys, PaddedPODArray * null_map) const; + + Block getSampleBlock(const Names &) const override; + + void checkTableCanBeRenamed(const StorageID & new_name) const override; + void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override; + + bool supportsParallelInsert() const override { return true; } + bool supportsIndexForIn() const override { return true; } + bool mayBenefitFromIndexForIn( + const ASTPtr & node, ContextPtr /*query_context*/, const StorageMetadataPtr & /*metadata_snapshot*/) const override + { + return node->getColumnName() == primary_key; + } + + zkutil::ZooKeeperPtr getClient() const; + const std::string & dataPath() const; + std::string fullPathForKey(std::string_view key) const; + + UInt64 keysLimit() const; + + template + void checkTable() const + { + auto is_table_valid = isTableValid(); + if (!is_table_valid.has_value()) + { + static constexpr std::string_view error_msg = "Failed to activate table because of connection issues. It will be activated " + "once a connection is established and metadata is verified"; + if constexpr (throw_on_error) + throw Exception(ErrorCodes::INVALID_STATE, error_msg); + else + { + LOG_ERROR(log, fmt::runtime(error_msg)); + return; + } + } + + if (!*is_table_valid) + { + static constexpr std::string_view error_msg + = "Failed to activate table because of invalid metadata in ZooKeeper. Please DETACH table"; + if constexpr (throw_on_error) + throw Exception(ErrorCodes::INVALID_STATE, error_msg); + else + { + LOG_ERROR(log, fmt::runtime(error_msg)); + return; + } + } + } + +private: + bool dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::EphemeralNodeHolder::Ptr & metadata_drop_lock); + + std::optional isTableValid() const; + + std::string root_path; + std::string primary_key; + + std::string data_path; + + std::string metadata_path; + + std::string tables_path; + std::string table_path; + + std::string dropped_path; + std::string dropped_lock_path; + + std::string zookeeper_name; + + std::string metadata_string; + + uint64_t keys_limit{0}; + + mutable std::mutex zookeeper_mutex; + mutable zkutil::ZooKeeperPtr zookeeper_client{nullptr}; + + mutable std::mutex init_mutex; + mutable std::optional table_is_valid; + + Poco::Logger * log; +}; + +} diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 5adc1974257..e4062734352 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -335,6 +335,13 @@ void StorageMergeTree::alter( mutation_version = startMutation(maybe_mutation_commands, local_context); } + { + /// Reset Object columns, because column of type + /// Object may be added or dropped by alter. + auto parts_lock = lockParts(); + resetObjectColumnsFromActiveParts(parts_lock); + } + /// Always execute required mutations synchronously, because alters /// should be executed in sequential order. if (!maybe_mutation_commands.empty()) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index c8eef26440e..28328b49f49 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3649,7 +3649,7 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, bool is_ if (quorum_entry.replicas.size() >= quorum_entry.required_number_of_replicas) { /// The quorum is reached. Delete the node, and update information about the last part that was successfully written with quorum. - LOG_TRACE(log, "Got {} (of {}) replicas confirmed quorum {}, going to remove node", + LOG_TRACE(log, "Got {} (of {} required) replicas confirmed quorum {}, going to remove node", quorum_entry.replicas.size(), quorum_entry.required_number_of_replicas, quorum_status_path); Coordination::Requests ops; @@ -4649,6 +4649,13 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer LOG_INFO(log, "Applied changes to the metadata of the table. Current metadata version: {}", metadata_version); } + { + /// Reset Object columns, because column of type + /// Object may be added or dropped by alter. + auto parts_lock = lockParts(); + resetObjectColumnsFromActiveParts(parts_lock); + } + /// This transaction may not happen, but it's OK, because on the next retry we will eventually create/update this node /// TODO Maybe do in in one transaction for Replicated database? zookeeper->createOrUpdate(fs::path(replica_path) / "metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent); @@ -7433,12 +7440,21 @@ std::unique_ptr StorageReplicatedMergeTree::getDefaultSetting String StorageReplicatedMergeTree::getTableSharedID() const { + /// Lock is not required in other places because createTableSharedID() + /// can be called only during table initialization + std::lock_guard lock(table_shared_id_mutex); + + /// Can happen if table was partially initialized before drop by DatabaseCatalog + if (table_shared_id == UUIDHelpers::Nil) + createTableSharedID(); + return toString(table_shared_id); } -void StorageReplicatedMergeTree::createTableSharedID() +void StorageReplicatedMergeTree::createTableSharedID() const { + LOG_DEBUG(log, "Creating shared ID for table {}", getStorageID().getNameForLogs()); if (table_shared_id != UUIDHelpers::Nil) throw Exception(ErrorCodes::LOGICAL_ERROR, "Table shared id already initialized"); @@ -7447,6 +7463,7 @@ void StorageReplicatedMergeTree::createTableSharedID() String id; if (!zookeeper->tryGet(zookeeper_table_id_path, id)) { + LOG_DEBUG(log, "Shared ID for table {} doesn't exist in ZooKeeper on path {}", getStorageID().getNameForLogs(), zookeeper_table_id_path); UUID table_id_candidate; auto local_storage_id = getStorageID(); if (local_storage_id.uuid != UUIDHelpers::Nil) @@ -7455,11 +7472,13 @@ void StorageReplicatedMergeTree::createTableSharedID() table_id_candidate = UUIDHelpers::generateV4(); id = toString(table_id_candidate); + LOG_DEBUG(log, "Got candidate ID {}, will try to create it in ZooKeeper on path {}", id, zookeeper_table_id_path); auto code = zookeeper->tryCreate(zookeeper_table_id_path, id, zkutil::CreateMode::Persistent); if (code == Coordination::Error::ZNODEEXISTS) { /// Other replica create node early id = zookeeper->get(zookeeper_table_id_path); + LOG_DEBUG(log, "Shared ID on path {} concurrently created, will set ID {}", zookeeper_table_id_path, id); } else if (code != Coordination::Error::ZOK) { @@ -7467,6 +7486,7 @@ void StorageReplicatedMergeTree::createTableSharedID() } } + LOG_DEBUG(log, "Initializing table shared ID with {}", id); table_shared_id = parseFromString(id); } @@ -7597,7 +7617,7 @@ std::pair StorageReplicatedMergeTree::unlockSharedData(const IMer } else { - LOG_TRACE(log, "Part {} looks temporary, because checksums file doesn't exists, blobs can be removed", part.name); + LOG_TRACE(log, "Part {} looks temporary, because {} file doesn't exists, blobs can be removed", part.name, IMergeTreeDataPart::FILE_FOR_REFERENCES_CHECK); /// Temporary part with some absent file cannot be locked in shared mode return std::make_pair(true, NameSet{}); } @@ -8222,25 +8242,12 @@ void StorageReplicatedMergeTree::createZeroCopyLockNode( } } -bool StorageReplicatedMergeTree::removeDetachedPart(DiskPtr disk, const String & path, const String & part_name, bool is_freezed) +bool StorageReplicatedMergeTree::removeDetachedPart(DiskPtr disk, const String & path, const String & part_name) { if (disk->supportZeroCopyReplication()) { - if (is_freezed) - { - FreezeMetaData meta; - if (meta.load(disk, path)) - { - FreezeMetaData::clean(disk, path); - return removeSharedDetachedPart(disk, path, part_name, meta.table_shared_id, meta.zookeeper_name, meta.replica_name, "", getContext()); - } - } - else - { - String table_id = getTableSharedID(); - - return removeSharedDetachedPart(disk, path, part_name, table_id, zookeeper_name, replica_name, zookeeper_path, getContext()); - } + String table_id = getTableSharedID(); + return removeSharedDetachedPart(disk, path, part_name, table_id, zookeeper_name, replica_name, zookeeper_path, getContext(), current_zookeeper); } disk->removeRecursive(path); @@ -8250,11 +8257,10 @@ bool StorageReplicatedMergeTree::removeDetachedPart(DiskPtr disk, const String & bool StorageReplicatedMergeTree::removeSharedDetachedPart(DiskPtr disk, const String & path, const String & part_name, const String & table_uuid, - const String &, const String & detached_replica_name, const String & detached_zookeeper_path, ContextPtr local_context) + const String &, const String & detached_replica_name, const String & detached_zookeeper_path, ContextPtr local_context, const zkutil::ZooKeeperPtr & zookeeper) { bool keep_shared = false; - zkutil::ZooKeeperPtr zookeeper = local_context->getZooKeeper(); NameSet files_not_to_remove; fs::path checksums = fs::path(path) / IMergeTreeDataPart::FILE_FOR_REFERENCES_CHECK; diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 6738dc1b72e..acf30112f76 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -325,7 +325,7 @@ public: void checkBrokenDisks(); static bool removeSharedDetachedPart(DiskPtr disk, const String & path, const String & part_name, const String & table_uuid, - const String & zookeeper_name, const String & replica_name, const String & zookeeper_path, ContextPtr local_context); + const String & zookeeper_name, const String & replica_name, const String & zookeeper_path, ContextPtr local_context, const zkutil::ZooKeeperPtr & zookeeper); bool canUseZeroCopyReplication() const; private: @@ -475,7 +475,8 @@ private: ThrottlerPtr replicated_sends_throttler; /// Global ID, synced via ZooKeeper between replicas - UUID table_shared_id; + mutable std::mutex table_shared_id_mutex; + mutable UUID table_shared_id; std::mutex last_broken_disks_mutex; std::set last_broken_disks; @@ -834,13 +835,13 @@ private: int32_t mode = zkutil::CreateMode::Persistent, bool replace_existing_lock = false, const String & path_to_set_hardlinked_files = "", const NameSet & hardlinked_files = {}); - bool removeDetachedPart(DiskPtr disk, const String & path, const String & part_name, bool is_freezed) override; + bool removeDetachedPart(DiskPtr disk, const String & path, const String & part_name) override; /// Create freeze metadata for table and save in zookeeper. Required only if zero-copy replication enabled. void createAndStoreFreezeMetadata(DiskPtr disk, DataPartPtr part, String backup_part_path) const override; // Create table id if needed - void createTableSharedID(); + void createTableSharedID() const; bool checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk); diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 627679d6779..ad91a30aa82 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1,4 +1,5 @@ #include +#include #include "IO/ParallelReadBuffer.h" #include "IO/IOThreadPool.h" #include "Parsers/ASTCreateQuery.h" @@ -63,6 +64,12 @@ namespace fs = std::filesystem; static const String PARTITION_ID_WILDCARD = "{_partition_id}"; +namespace ProfileEvents +{ + extern const Event S3DeleteObjects; + extern const Event S3ListObjects; +} + namespace DB { @@ -164,6 +171,7 @@ private: { buffer.clear(); + ProfileEvents::increment(ProfileEvents::S3ListObjects); outcome = client.ListObjectsV2(request); if (!outcome.IsSuccess()) throw Exception(ErrorCodes::S3_ERROR, "Could not list objects in bucket {} with prefix {}, S3 exception: {}, message: {}", @@ -475,7 +483,7 @@ std::unique_ptr StorageS3Source::createS3ReadBuffer(const String & k if (it != object_infos.end()) object_size = it->second.size; else - object_size = DB::S3::getObjectSize(client, bucket, key, version_id, false); + object_size = DB::S3::getObjectSize(client, bucket, key, version_id, false, false); auto download_buffer_size = getContext()->getSettings().max_download_buffer_size; const bool use_parallel_download = download_buffer_size > 0 && download_thread_num > 1; @@ -559,6 +567,7 @@ static bool checkIfObjectExists(const std::shared_ptr & request.SetPrefix(key); while (!is_finished) { + ProfileEvents::increment(ProfileEvents::S3ListObjects); outcome = client->ListObjectsV2(request); if (!outcome.IsSuccess()) throw Exception( @@ -1036,6 +1045,7 @@ void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, delkeys.AddObjects(std::move(obj)); } + ProfileEvents::increment(ProfileEvents::S3DeleteObjects); Aws::S3::Model::DeleteObjectsRequest request; request.SetBucket(s3_configuration.uri.bucket); request.SetDelete(delkeys); @@ -1379,7 +1389,7 @@ std::optional StorageS3::tryGetColumnsFromCache( /// Note that in case of exception in getObjectInfo returned info will be empty, /// but schema cache will handle this case and won't return columns from cache /// because we can't say that it's valid without last modification time. - info = S3::getObjectInfo(s3_configuration.client, s3_configuration.uri.bucket, *it, s3_configuration.uri.version_id, false); + info = S3::getObjectInfo(s3_configuration.client, s3_configuration.uri.bucket, *it, s3_configuration.uri.version_id, false, false); if (object_infos) (*object_infos)[path] = info; } diff --git a/src/Storages/StorageS3Settings.cpp b/src/Storages/StorageS3Settings.cpp index 353e324c853..4ab3375e188 100644 --- a/src/Storages/StorageS3Settings.cpp +++ b/src/Storages/StorageS3Settings.cpp @@ -121,6 +121,7 @@ S3Settings::ReadWriteSettings::ReadWriteSettings(const Settings & settings) max_single_part_upload_size = settings.s3_max_single_part_upload_size; max_connections = settings.s3_max_connections; check_objects_after_upload = settings.s3_check_objects_after_upload; + max_unexpected_write_error_retries = settings.s3_max_unexpected_write_error_retries; } void S3Settings::ReadWriteSettings::updateFromSettingsIfEmpty(const Settings & settings) @@ -137,6 +138,8 @@ void S3Settings::ReadWriteSettings::updateFromSettingsIfEmpty(const Settings & s max_single_part_upload_size = settings.s3_max_single_part_upload_size; if (!max_connections) max_connections = settings.s3_max_connections; + if (!max_unexpected_write_error_retries) + max_unexpected_write_error_retries = settings.s3_max_unexpected_write_error_retries; check_objects_after_upload = settings.s3_check_objects_after_upload; } diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index 9ef51c77191..41136117b24 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -61,6 +61,7 @@ struct S3Settings size_t max_single_part_upload_size = 0; size_t max_connections = 0; bool check_objects_after_upload = false; + size_t max_unexpected_write_error_retries = 0; ReadWriteSettings() = default; explicit ReadWriteSettings(const Settings & settings); @@ -73,7 +74,8 @@ struct S3Settings && upload_part_size_multiply_parts_count_threshold == other.upload_part_size_multiply_parts_count_threshold && max_single_part_upload_size == other.max_single_part_upload_size && max_connections == other.max_connections - && check_objects_after_upload == other.check_objects_after_upload; + && check_objects_after_upload == other.check_objects_after_upload + && max_unexpected_write_error_retries == other.max_unexpected_write_error_retries; } void updateFromSettingsIfEmpty(const Settings & settings); diff --git a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp index 111ea343398..67867b6c577 100644 --- a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp +++ b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp @@ -205,9 +205,9 @@ static void fillStatusColumns(MutableColumns & res_columns, size_t & col, void StorageSystemDDLWorkerQueue::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const { - zkutil::ZooKeeperPtr zookeeper = context->getZooKeeper(); - fs::path ddl_zookeeper_path = context->getConfigRef().getString("distributed_ddl.path", "/clickhouse/task_queue/ddl/"); - + auto& ddl_worker = context->getDDLWorker(); + fs::path ddl_zookeeper_path = ddl_worker.getQueueDir(); + zkutil::ZooKeeperPtr zookeeper = ddl_worker.getAndSetZooKeeper(); Strings ddl_task_paths = zookeeper->getChildren(ddl_zookeeper_path); GetResponseFutures ddl_task_futures; diff --git a/src/Storages/System/StorageSystemModels.cpp b/src/Storages/System/StorageSystemModels.cpp index 4a4dbbc69df..d06f97a3f54 100644 --- a/src/Storages/System/StorageSystemModels.cpp +++ b/src/Storages/System/StorageSystemModels.cpp @@ -1,11 +1,11 @@ #include +#include #include #include #include #include #include -#include -#include +#include namespace DB @@ -14,45 +14,24 @@ namespace DB NamesAndTypesList StorageSystemModels::getNamesAndTypes() { return { - { "name", std::make_shared() }, - { "status", std::make_shared(getStatusEnumAllPossibleValues()) }, - { "origin", std::make_shared() }, + { "model_path", std::make_shared() }, { "type", std::make_shared() }, { "loading_start_time", std::make_shared() }, { "loading_duration", std::make_shared() }, - //{ "creation_time", std::make_shared() }, - { "last_exception", std::make_shared() }, }; } void StorageSystemModels::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const { - const auto & external_models_loader = context->getExternalModelsLoader(); - auto load_results = external_models_loader.getLoadResults(); + auto bridge_helper = std::make_unique(context); + ExternalModelInfos infos = bridge_helper->listModels(); - for (const auto & load_result : load_results) + for (const auto & info : infos) { - res_columns[0]->insert(load_result.name); - res_columns[1]->insert(static_cast(load_result.status)); - res_columns[2]->insert(load_result.config ? load_result.config->path : ""); - - if (load_result.object) - { - const auto model_ptr = std::static_pointer_cast(load_result.object); - res_columns[3]->insert(model_ptr->getTypeName()); - } - else - { - res_columns[3]->insertDefault(); - } - - res_columns[4]->insert(static_cast(std::chrono::system_clock::to_time_t(load_result.loading_start_time))); - res_columns[5]->insert(std::chrono::duration_cast>(load_result.loading_duration).count()); - - if (load_result.exception) - res_columns[6]->insert(getExceptionMessage(load_result.exception, false)); - else - res_columns[6]->insertDefault(); + res_columns[0]->insert(info.model_path); + res_columns[1]->insert(info.model_type); + res_columns[2]->insert(static_cast(std::chrono::system_clock::to_time_t(info.loading_start_time))); + res_columns[3]->insert(std::chrono::duration_cast>(info.loading_duration).count()); } } diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index 07f3cd0ccc5..d788efd8860 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -90,6 +90,8 @@ StorageSystemParts::StorageSystemParts(const StorageID & table_id_) {"removal_tid", getTransactionIDDataType()}, {"creation_csn", std::make_shared()}, {"removal_csn", std::make_shared()}, + + {"has_lightweight_delete", std::make_shared()}, } ) { @@ -305,6 +307,8 @@ void StorageSystemParts::processNextStorage( columns[res_index++]->insert(part->version.creation_csn.load(std::memory_order_relaxed)); if (columns_mask[src_index++]) columns[res_index++]->insert(part->version.removal_csn.load(std::memory_order_relaxed)); + if (columns_mask[src_index++]) + columns[res_index++]->insert(part->hasLightweightDelete()); /// _state column should be the latest. /// Do not use part->getState*, it can be changed from different thread diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index e1f1e4985b4..f0ee90531c9 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -40,8 +40,8 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[3]->insert(setting.getDescription()); Field min, max; - bool read_only = false; - constraints.get(setting_name, min, max, read_only); + SettingConstraintWritability writability = SettingConstraintWritability::WRITABLE; + constraints.get(settings, setting_name, min, max, writability); /// These two columns can accept strings only. if (!min.isNull()) @@ -49,17 +49,9 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co if (!max.isNull()) max = Settings::valueToStringUtil(setting_name, max); - if (!read_only) - { - if ((settings.readonly == 1) - || ((settings.readonly > 1) && (setting_name == "readonly")) - || ((!settings.allow_ddl) && (setting_name == "allow_ddl"))) - read_only = true; - } - res_columns[4]->insert(min); res_columns[5]->insert(max); - res_columns[6]->insert(read_only); + res_columns[6]->insert(writability == SettingConstraintWritability::CONST); res_columns[7]->insert(setting.getTypeName()); } } diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp index 565ff5e471e..6785a4392e1 100644 --- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -12,11 +12,24 @@ #include #include #include +#include namespace DB { +const std::vector> & getSettingConstraintWritabilityEnumValues() +{ + static const std::vector> values = [] + { + std::vector> res; + for (auto value : collections::range(SettingConstraintWritability::MAX)) + res.emplace_back(toString(value), static_cast(value)); + return res; + }(); + return values; +} + NamesAndTypesList StorageSystemSettingsProfileElements::getNamesAndTypes() { NamesAndTypesList names_and_types{ @@ -28,7 +41,7 @@ NamesAndTypesList StorageSystemSettingsProfileElements::getNamesAndTypes() {"value", std::make_shared(std::make_shared())}, {"min", std::make_shared(std::make_shared())}, {"max", std::make_shared(std::make_shared())}, - {"readonly", std::make_shared(std::make_shared())}, + {"writability", std::make_shared(std::make_shared(getSettingConstraintWritabilityEnumValues()))}, {"inherit_profile", std::make_shared(std::make_shared())}, }; return names_and_types; @@ -62,8 +75,8 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns auto & column_min_null_map = assert_cast(*res_columns[i++]).getNullMapData(); auto & column_max = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); auto & column_max_null_map = assert_cast(*res_columns[i++]).getNullMapData(); - auto & column_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); - auto & column_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_writability = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_writability_null_map = assert_cast(*res_columns[i++]).getNullMapData(); auto & column_inherit_profile = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); auto & column_inherit_profile_null_map = assert_cast(*res_columns[i++]).getNullMapData(); @@ -100,16 +113,16 @@ void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns inserted_max = true; } - bool inserted_readonly = false; - if (element.readonly && !element.setting_name.empty()) + bool inserted_writability = false; + if (element.writability && !element.setting_name.empty()) { - column_readonly.push_back(*element.readonly); - column_readonly_null_map.push_back(false); - inserted_readonly = true; + column_writability.insertValue(static_cast(*element.writability)); + column_writability_null_map.push_back(false); + inserted_writability = true; } bool inserted_setting_name = false; - if (inserted_value || inserted_min || inserted_max || inserted_readonly) + if (inserted_value || inserted_min || inserted_max || inserted_writability) { const auto & setting_name = element.setting_name; column_setting_name.insertData(setting_name.data(), setting_name.size()); diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index 575b3de7ae2..055270be4ae 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -88,6 +88,7 @@ void registerStorageFileLog(StorageFactory & factory); void registerStorageSQLite(StorageFactory & factory); #endif +void registerStorageKeeperMap(StorageFactory & factory); void registerStorages() { @@ -171,6 +172,8 @@ void registerStorages() #if USE_SQLITE registerStorageSQLite(factory); #endif + + registerStorageKeeperMap(factory); } } diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index 3fa2f93b484..f1079a9ee10 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -5,8 +5,6 @@ #include #include #include -#include -#include #include #include #include @@ -18,18 +16,12 @@ #include #include #include -#include #include #include #include #include #include -#if !defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wsuggest-override" -#endif - DB::StoragePtr createStorage(DB::DiskPtr & disk) { diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index f58c7a74dfe..d668dbe0498 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -291,7 +291,9 @@ def main(): logging.info("Will try to fetch cache for our build") try: - get_ccache_if_not_exists(ccache_path, s3_helper, pr_info.number, TEMP_PATH) + get_ccache_if_not_exists( + ccache_path, s3_helper, pr_info.number, TEMP_PATH, pr_info.release_pr + ) except Exception as e: # In case there are issues with ccache, remove the path and do not fail a build logging.info("Failed to get ccache, building without it. Error: %s", e) diff --git a/tests/ci/ccache_utils.py b/tests/ci/ccache_utils.py index cfe07363589..864b3a8f9b6 100644 --- a/tests/ci/ccache_utils.py +++ b/tests/ci/ccache_utils.py @@ -11,6 +11,7 @@ import requests # type: ignore from compress_files import decompress_fast, compress_fast from env_helper import S3_DOWNLOAD, S3_BUILDS_BUCKET +from s3_helper import S3Helper DOWNLOAD_RETRIES_COUNT = 5 @@ -57,12 +58,19 @@ def dowload_file_with_progress(url, path): def get_ccache_if_not_exists( - path_to_ccache_dir, s3_helper, current_pr_number, temp_path + path_to_ccache_dir: str, + s3_helper: S3Helper, + current_pr_number: int, + temp_path: str, + release_pr: int, ) -> int: """returns: number of PR for downloaded PR. -1 if ccache not found""" ccache_name = os.path.basename(path_to_ccache_dir) cache_found = False prs_to_check = [current_pr_number] + # Release PR is either 0 or defined + if release_pr: + prs_to_check.append(release_pr) ccache_pr = -1 if current_pr_number != 0: prs_to_check.append(0) diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py index 064a0b3add1..d1c9d3d394c 100644 --- a/tests/ci/cherry_pick.py +++ b/tests/ci/cherry_pick.py @@ -44,11 +44,11 @@ from ssh import SSHKey class Labels: - LABEL_MUST_BACKPORT = "pr-must-backport" - LABEL_BACKPORT = "pr-backport" - LABEL_BACKPORTED = "pr-backported" - LABEL_CHERRYPICK = "pr-cherrypick" - LABEL_DO_NOT_TEST = "do not test" + MUST_BACKPORT = "pr-must-backport" + BACKPORT = "pr-backport" + BACKPORTS_CREATED = "pr-backports-created" + CHERRYPICK = "pr-cherrypick" + DO_NOT_TEST = "do not test" class ReleaseBranch: @@ -204,8 +204,8 @@ Merge it only if you intend to backport changes to the target branch, otherwise base=self.backport_branch, head=self.cherrypick_branch, ) - self.cherrypick_pr.add_to_labels(Labels.LABEL_CHERRYPICK) - self.cherrypick_pr.add_to_labels(Labels.LABEL_DO_NOT_TEST) + self.cherrypick_pr.add_to_labels(Labels.CHERRYPICK) + self.cherrypick_pr.add_to_labels(Labels.DO_NOT_TEST) self._assign_new_pr(self.cherrypick_pr) def create_backport(self): @@ -236,7 +236,7 @@ Merge it only if you intend to backport changes to the target branch, otherwise base=self.name, head=self.backport_branch, ) - self.backport_pr.add_to_labels(Labels.LABEL_BACKPORT) + self.backport_pr.add_to_labels(Labels.BACKPORT) self._assign_new_pr(self.backport_pr) def _assign_new_pr(self, new_pr: PullRequest): @@ -321,8 +321,8 @@ class Backport: tomorrow = date.today() + timedelta(days=1) logging.info("Receive PRs suppose to be backported") self.prs_for_backport = self.gh.get_pulls_from_search( - query=f"{self._query} -label:pr-backported", - label=",".join(self.labels_to_backport + [Labels.LABEL_MUST_BACKPORT]), + query=f"{self._query} -label:{Labels.BACKPORTS_CREATED}", + label=",".join(self.labels_to_backport + [Labels.MUST_BACKPORT]), merged=[since_date, tomorrow], ) logging.info( @@ -342,7 +342,7 @@ class Backport: def process_pr(self, pr: PullRequest): pr_labels = [label.name for label in pr.labels] - if Labels.LABEL_MUST_BACKPORT in pr_labels: + if Labels.MUST_BACKPORT in pr_labels: branches = [ ReleaseBranch(br, pr) for br in self.release_branches ] # type: List[ReleaseBranch] @@ -407,11 +407,11 @@ class Backport: if self.dry_run: logging.info("DRY RUN: would mark PR #%s as done", pr.number) return - pr.add_to_labels(Labels.LABEL_BACKPORTED) + pr.add_to_labels(Labels.BACKPORTS_CREATED) logging.info( "PR #%s is successfully labeled with `%s`", pr.number, - Labels.LABEL_BACKPORTED, + Labels.BACKPORTS_CREATED, ) @property diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index fa68d1982d2..a31f2298a58 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -8,7 +8,7 @@ BuildConfig = Dict[str, ConfValue] CI_CONFIG = { "build_config": { "package_release": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "", "package_type": "deb", @@ -19,7 +19,7 @@ CI_CONFIG = { "with_coverage": False, }, "coverity": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "", "package_type": "coverity", @@ -29,7 +29,7 @@ CI_CONFIG = { "official": False, }, "package_aarch64": { - "compiler": "clang-14-aarch64", + "compiler": "clang-15-aarch64", "build_type": "", "sanitizer": "", "package_type": "deb", @@ -40,7 +40,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_asan": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "address", "package_type": "deb", @@ -49,7 +49,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_ubsan": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "undefined", "package_type": "deb", @@ -67,7 +67,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_msan": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "memory", "package_type": "deb", @@ -76,7 +76,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_debug": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "debug", "sanitizer": "", "package_type": "deb", @@ -85,7 +85,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_release": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -94,7 +94,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_tidy": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "debug", "sanitizer": "", "package_type": "binary", @@ -104,7 +104,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_shared": { - "compiler": "clang-14", + "compiler": "clang-15", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -113,7 +113,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_darwin": { - "compiler": "clang-14-darwin", + "compiler": "clang-15-darwin", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -123,7 +123,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_aarch64": { - "compiler": "clang-14-aarch64", + "compiler": "clang-15-aarch64", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -132,7 +132,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_freebsd": { - "compiler": "clang-14-freebsd", + "compiler": "clang-15-freebsd", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -142,7 +142,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_darwin_aarch64": { - "compiler": "clang-14-darwin-aarch64", + "compiler": "clang-15-darwin-aarch64", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -152,7 +152,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_ppc64le": { - "compiler": "clang-14-ppc64le", + "compiler": "clang-15-ppc64le", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -162,7 +162,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_amd64sse2": { - "compiler": "clang-14-amd64sse2", + "compiler": "clang-15-amd64sse2", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -342,7 +342,7 @@ CI_CONFIG = { }, "Performance Comparison Aarch64": { "required_build": "package_aarch64", - "test_grep_exclude_filter": "constant_column_search", + "test_grep_exclude_filter": "", }, }, } # type: dict diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 181555f3a94..773f3ac1b57 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -164,7 +164,7 @@ def gen_versions( # The order is important, PR number is used as cache during the build versions = [str(pr_info.number), pr_commit_version] result_version = pr_commit_version - if pr_info.number == 0: + if pr_info.number == 0 and pr_info.base_name == "master": # First get the latest for cache versions.insert(0, "latest") diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index 32df6d5f1d0..740cae5bc97 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -99,6 +99,11 @@ class TestDockerImageCheck(unittest.TestCase): def test_gen_version(self): pr_info = PRInfo(PRInfo.default_event.copy()) + pr_info.base_name = "anything-else" + versions, result_version = di.gen_versions(pr_info, None) + self.assertEqual(versions, ["0", "0-HEAD"]) + self.assertEqual(result_version, "0-HEAD") + pr_info.base_name = "master" versions, result_version = di.gen_versions(pr_info, None) self.assertEqual(versions, ["latest", "0", "0-HEAD"]) self.assertEqual(result_version, "0-HEAD") diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 038289406de..03e42726808 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -125,7 +125,7 @@ if __name__ == "__main__": logging.info("Will try to fetch cache for our build") ccache_for_pr = get_ccache_if_not_exists( - cache_path, s3_helper, pr_info.number, temp_path + cache_path, s3_helper, pr_info.number, temp_path, pr_info.release_pr ) upload_master_ccache = ccache_for_pr in (-1, 0) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 2acd0e4c811..77421ddac32 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -86,7 +86,7 @@ class PRInfo: self.changed_files = set() # type: Set[str] self.body = "" self.diff_urls = [] - self.release_pr = "" + self.release_pr = 0 ref = github_event.get("ref", "refs/head/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh index 3dfd1761e88..66a38a6a37d 100644 --- a/tests/ci/worker/init_runner.sh +++ b/tests/ci/worker/init_runner.sh @@ -76,13 +76,13 @@ if [[ ${ROOT_STAT[0]} -lt 3000000 ]] || [[ ${ROOT_STAT[1]} -lt 5 ]]; then fi # shellcheck disable=SC2046 -docker kill $(docker ps -q) ||: +docker ps --quiet | xargs --no-run-if-empty docker kill ||: # shellcheck disable=SC2046 -docker rm -f $(docker ps -a -q) ||: +docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # If we have hanged containers after the previous commands, than we have a hanged one # and should restart the daemon -if [ "$(docker ps -a -q)" ]; then +if [ "$(docker ps --all --quiet)" ]; then # Systemd service of docker has StartLimitBurst=3 and StartLimitInterval=60s, # that's why we try restarting it for long for i in {1..25}; diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 14cf4d0674a..b0b03daf3b5 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1808,6 +1808,9 @@ def main(args): args, "system", "processes", "is_all_data_sent" ) + if args.s3_storage and (BuildFlags.THREAD in args.build_flags or BuildFlags.DEBUG in args.build_flags): + args.no_random_settings = True + if args.skip: args.skip = set(args.skip) diff --git a/tests/config/config.d/enable_access_control_improvements.xml b/tests/config/config.d/enable_access_control_improvements.xml index 5a186548098..564b656a0ad 100644 --- a/tests/config/config.d/enable_access_control_improvements.xml +++ b/tests/config/config.d/enable_access_control_improvements.xml @@ -4,5 +4,6 @@ true true true + true diff --git a/tests/config/config.d/enable_keeper_map.xml b/tests/config/config.d/enable_keeper_map.xml new file mode 100644 index 00000000000..b4cbb6a954b --- /dev/null +++ b/tests/config/config.d/enable_keeper_map.xml @@ -0,0 +1,3 @@ + + /test_keeper_map + diff --git a/tests/config/config.d/system_unfreeze.xml b/tests/config/config.d/system_unfreeze.xml new file mode 100644 index 00000000000..9d6e7279e09 --- /dev/null +++ b/tests/config/config.d/system_unfreeze.xml @@ -0,0 +1,4 @@ + + + true + diff --git a/tests/config/install.sh b/tests/config/install.sh index e27675b8abb..d4c71212423 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -47,8 +47,10 @@ ln -sf $SRC_PATH/config.d/named_collection.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/ssl_certs.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/filesystem_cache_log.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/session_log.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/system_unfreeze.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/enable_zero_copy_replication.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/nlp.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/enable_keeper_map.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/users.d/log_queries.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/readonly.xml $DEST_SERVER_PATH/users.d/ @@ -61,6 +63,7 @@ ln -sf $SRC_PATH/users.d/memory_profiler.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/no_fsync_metadata.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/filelog.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/enable_blobs_check.xml $DEST_SERVER_PATH/users.d/ +ln -sf $SRC_PATH/users.d/marks.xml $DEST_SERVER_PATH/users.d/ # FIXME DataPartsExchange may hang for http_send_timeout seconds # when nobody is going to read from the other side of socket (due to "Fetching of part was cancelled"), diff --git a/tests/config/users.d/marks.xml b/tests/config/users.d/marks.xml new file mode 100644 index 00000000000..8a311c4affe --- /dev/null +++ b/tests/config/users.d/marks.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/fuzz/all.dict b/tests/fuzz/all.dict index dff62cd68a7..a147878da9b 100644 --- a/tests/fuzz/all.dict +++ b/tests/fuzz/all.dict @@ -763,7 +763,6 @@ "MINUTE" "MM" "mod" -"modelEvaluate" "MODIFY" "MODIFY COLUMN" "MODIFY ORDER BY" diff --git a/tests/fuzz/dictionaries/functions.dict b/tests/fuzz/dictionaries/functions.dict index cbcad3c05da..b90697f0c3d 100644 --- a/tests/fuzz/dictionaries/functions.dict +++ b/tests/fuzz/dictionaries/functions.dict @@ -469,7 +469,6 @@ "subtractSeconds" "alphaTokens" "negate" -"modelEvaluate" "file" "roundAge" "MACStringToOUI" diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 562497fe8b0..4eab305358b 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -168,7 +168,8 @@ def clear_ip_tables_and_restart_daemons(): try: logging.info("Killing all alive docker containers") subprocess.check_output( - "timeout -s 9 10m docker kill $(docker ps -q)", shell=True + "timeout -s 9 10m docker ps --quiet | xargs --no-run-if-empty docker kill", + shell=True, ) except subprocess.CalledProcessError as err: logging.info("docker kill excepted: " + str(err)) @@ -176,7 +177,8 @@ def clear_ip_tables_and_restart_daemons(): try: logging.info("Removing all docker containers") subprocess.check_output( - "timeout -s 9 10m docker rm $(docker ps -a -q) --force", shell=True + "timeout -s 9 10m docker ps --all --quiet | xargs --no-run-if-empty docker rm --force", + shell=True, ) except subprocess.CalledProcessError as err: logging.info("docker rm excepted: " + str(err)) diff --git a/tests/integration/helpers/0_common_instance_config.xml b/tests/integration/helpers/0_common_instance_config.xml index 64f0ce9e361..27563e47c35 100644 --- a/tests/integration/helpers/0_common_instance_config.xml +++ b/tests/integration/helpers/0_common_instance_config.xml @@ -24,5 +24,6 @@ true true true + true diff --git a/tests/integration/runner b/tests/integration/runner index e1b9a55b43e..f4f853e00ad 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -135,7 +135,7 @@ def check_args_and_update_paths(args): def docker_kill_handler_handler(signum, frame): subprocess.check_call( - 'docker kill $(docker ps -a -q --filter name={name} --format="{{{{.ID}}}}")'.format( + 'docker ps --all --quiet --filter name={name} --format="{{{{.ID}}}}"'.format( name=CONTAINER_NAME ), shell=True, @@ -404,7 +404,7 @@ if __name__ == "__main__": ) containers = subprocess.check_output( - f"docker ps -a -q --filter name={CONTAINER_NAME} --format={{{{.ID}}}}", + f"docker ps --all --quiet --filter name={CONTAINER_NAME} --format={{{{.ID}}}}", shell=True, universal_newlines=True, ).splitlines() diff --git a/tests/integration/test_catboost_model_config_reload/__init__.py b/tests/integration/test_catboost_evaluate/__init__.py similarity index 100% rename from tests/integration/test_catboost_model_config_reload/__init__.py rename to tests/integration/test_catboost_evaluate/__init__.py diff --git a/tests/integration/test_catboost_evaluate/config/models_config.xml b/tests/integration/test_catboost_evaluate/config/models_config.xml new file mode 100644 index 00000000000..f63df06ee26 --- /dev/null +++ b/tests/integration/test_catboost_evaluate/config/models_config.xml @@ -0,0 +1,3 @@ + + /etc/clickhouse-server/model/libcatboostmodel.so + diff --git a/tests/integration/test_catboost_evaluate/model/amazon_model.bin b/tests/integration/test_catboost_evaluate/model/amazon_model.bin new file mode 100644 index 00000000000..4a37fbec310 Binary files /dev/null and b/tests/integration/test_catboost_evaluate/model/amazon_model.bin differ diff --git a/tests/integration/test_catboost_model_config_reload/model/libcatboostmodel.so b/tests/integration/test_catboost_evaluate/model/libcatboostmodel.so similarity index 100% rename from tests/integration/test_catboost_model_config_reload/model/libcatboostmodel.so rename to tests/integration/test_catboost_evaluate/model/libcatboostmodel.so diff --git a/tests/integration/test_catboost_model_config_reload/model/model.bin b/tests/integration/test_catboost_evaluate/model/simple_model.bin similarity index 100% rename from tests/integration/test_catboost_model_config_reload/model/model.bin rename to tests/integration/test_catboost_evaluate/model/simple_model.bin diff --git a/tests/integration/test_catboost_evaluate/test.py b/tests/integration/test_catboost_evaluate/test.py new file mode 100644 index 00000000000..a0915977ab6 --- /dev/null +++ b/tests/integration/test_catboost_evaluate/test.py @@ -0,0 +1,402 @@ +import os +import sys +import time + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +instance = cluster.add_instance( + "instance", stay_alive=True, main_configs=["config/models_config.xml"] +) + + +@pytest.fixture(scope="module") +def ch_cluster(): + try: + cluster.start() + + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=os.path.join(SCRIPT_DIR, "model/."), + cont_id=instance.docker_id, + dist="/etc/clickhouse-server/model", + ) + ) + instance.restart_clickhouse() + + yield cluster + + finally: + cluster.shutdown() + + +# --------------------------------------------------------------------------- +# simple_model.bin has 2 float features and 9 categorical features + + +def testConstantFeatures(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + expected = "-1.930268705869267\n" + assert result == expected + + +def testNonConstantFeatures(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + instance.query("DROP TABLE IF EXISTS T;") + instance.query( + "CREATE TABLE T(ID UInt32, F1 Float32, F2 Float32, F3 UInt32, F4 UInt32, F5 UInt32, F6 UInt32, F7 UInt32, F8 UInt32, F9 Float32, F10 Float32, F11 Float32) ENGINE MergeTree ORDER BY ID;" + ) + instance.query("INSERT INTO T VALUES(0, 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);") + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11) from T;" + ) + expected = "-1.930268705869267\n" + assert result == expected + + instance.query("DROP TABLE IF EXISTS T;") + + +def testModelPathIsNotAConstString(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + err = instance.query_and_get_error( + "select catboostEvaluate(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert ( + "Illegal type UInt8 of first argument of function catboostEvaluate, expected a string" + in err + ) + + instance.query("DROP TABLE IF EXISTS T;") + instance.query("CREATE TABLE T(ID UInt32, A String) ENGINE MergeTree ORDER BY ID") + instance.query("INSERT INTO T VALUES(0, 'test');") + err = instance.query_and_get_error( + "select catboostEvaluate(A, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) FROM T;" + ) + assert ( + "First argument of function catboostEvaluate must be a constant string" in err + ) + instance.query("DROP TABLE IF EXISTS T;") + + +def testWrongNumberOfFeatureArguments(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin');" + ) + assert "Function catboostEvaluate expects at least 2 arguments" in err + + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1, 2);" + ) + assert ( + "Number of columns is different with number of features: columns size 2 float features size 2 + cat features size 9" + in err + ) + + +def testFloatFeatureMustBeNumeric(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 'a', 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert "Column 1 should be numeric to make float feature" in err + + +def testCategoricalFeatureMustBeNumericOrString(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, tuple(8), 9, 10, 11);" + ) + assert "Column 7 should be numeric or string" in err + + +def testOnLowCardinalityFeatures(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + # same but on domain-compressed data + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', toLowCardinality(1.0), toLowCardinality(2.0), toLowCardinality(3), toLowCardinality(4), toLowCardinality(5), toLowCardinality(6), toLowCardinality(7), toLowCardinality(8), toLowCardinality(9), toLowCardinality(10), toLowCardinality(11));" + ) + expected = "-1.930268705869267\n" + assert result == expected + + +def testOnNullableFeatures(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', toNullable(1.0), toNullable(2.0), toNullable(3), toNullable(4), toNullable(5), toNullable(6), toNullable(7), toNullable(8), toNullable(9), toNullable(10), toNullable(11));" + ) + expected = "-1.930268705869267\n" + assert result == expected + + # Actual NULLs are disallowed + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL), toNullable(NULL));" + ) + assert "Column 0 should be numeric to make float feature" in err + + +def testInvalidLibraryPath(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + # temporarily move library elsewhere + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/libcatboostmodel.so /etc/clickhouse-server/model/nonexistant.so", + ] + ) + + err = instance.query_and_get_error( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert ( + "Can't load library /etc/clickhouse-server/model/libcatboostmodel.so: file doesn't exist" + in err + ) + + # restore + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/nonexistant.so /etc/clickhouse-server/model/libcatboostmodel.so", + ] + ) + + +def testInvalidModelPath(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + err = instance.query_and_get_error( + "select catboostEvaluate('', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert "Can't load model : file doesn't exist" in err + + err = instance.query_and_get_error( + "select catboostEvaluate('model_non_existant.bin', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert "Can't load model model_non_existant.bin: file doesn't exist" in err + + +def testRecoveryAfterCrash(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + expected = "-1.930268705869267\n" + assert result == expected + + instance.exec_in_container( + ["bash", "-c", "kill -9 `pidof clickhouse-library-bridge`"], user="root" + ) + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + assert result == expected + + +# --------------------------------------------------------------------------- +# amazon_model.bin has 0 float features and 9 categorical features + + +def testAmazonModelSingleRow(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/amazon_model.bin', 1, 2, 3, 4, 5, 6, 7, 8, 9);" + ) + expected = "0.7774665009089274\n" + assert result == expected + + +def testAmazonModelManyRows(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + result = instance.query("drop table if exists amazon") + + result = instance.query( + "create table amazon ( DATE Date materialized today(), ACTION UInt8, RESOURCE UInt32, MGR_ID UInt32, ROLE_ROLLUP_1 UInt32, ROLE_ROLLUP_2 UInt32, ROLE_DEPTNAME UInt32, ROLE_TITLE UInt32, ROLE_FAMILY_DESC UInt32, ROLE_FAMILY UInt32, ROLE_CODE UInt32) engine = MergeTree order by DATE" + ) + + result = instance.query( + "insert into amazon select number % 256, number, number, number, number, number, number, number, number, number from numbers(7500)" + ) + + # First compute prediction, then as a very crude way to fingerprint and compare the result: sum and floor + # (the focus is to test that the exchange of large result sets between the server and the bridge works) + result = instance.query( + "SELECT floor(sum(catboostEvaluate('/etc/clickhouse-server/model/amazon_model.bin', RESOURCE, MGR_ID, ROLE_ROLLUP_1, ROLE_ROLLUP_2, ROLE_DEPTNAME, ROLE_TITLE, ROLE_FAMILY_DESC, ROLE_FAMILY, ROLE_CODE))) FROM amazon" + ) + + expected = "5834\n" + assert result == expected + + result = instance.query("drop table if exists amazon") + + +def testModelUpdate(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + query = "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + + result = instance.query(query) + expected = "-1.930268705869267\n" + assert result == expected + + # simulate an update of the model: temporarily move the amazon model in place of the simple model + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/simple_model.bin /etc/clickhouse-server/model/simple_model.bin.bak", + ] + ) + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/amazon_model.bin /etc/clickhouse-server/model/simple_model.bin", + ] + ) + + # unload simple model + result = instance.query( + "system reload model '/etc/clickhouse-server/model/simple_model.bin'" + ) + + # load the simple-model-camouflaged amazon model + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1, 2, 3, 4, 5, 6, 7, 8, 9);" + ) + expected = "0.7774665009089274\n" + assert result == expected + + # restore + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/simple_model.bin /etc/clickhouse-server/model/amazon_model.bin", + ] + ) + instance.exec_in_container( + [ + "bash", + "-c", + "mv /etc/clickhouse-server/model/simple_model.bin.bak /etc/clickhouse-server/model/simple_model.bin", + ] + ) + + +def testSystemModelsAndModelRefresh(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + result = instance.query("system reload models") + + # check model system view + result = instance.query("select * from system.models") + expected = "" + assert result == expected + + # load simple model + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/simple_model.bin', 1.0, 2.0, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) + expected = "-1.930268705869267\n" + assert result == expected + + # check model system view with one model loaded + result = instance.query("select * from system.models") + assert result.count("\n") == 1 + expected = "/etc/clickhouse-server/model/simple_model.bin" + assert expected in result + + # load amazon model + result = instance.query( + "select catboostEvaluate('/etc/clickhouse-server/model/amazon_model.bin', 1, 2, 3, 4, 5, 6, 7, 8, 9);" + ) + expected = "0.7774665009089274\n" + assert result == expected + + # check model system view with one model loaded + result = instance.query("select * from system.models") + assert result.count("\n") == 2 + expected = "/etc/clickhouse-server/model/simple_model.bin" + assert expected in result + expected = "/etc/clickhouse-server/model/amazon_model.bin" + assert expected in result + + # unload simple model + result = instance.query( + "system reload model '/etc/clickhouse-server/model/simple_model.bin'" + ) + + # check model system view, it should not display the removed model + result = instance.query("select * from system.models") + assert result.count("\n") == 1 + expected = "/etc/clickhouse-server/model/amazon_model.bin" + assert expected in result diff --git a/tests/integration/test_catboost_model_config_reload/config/catboost_lib.xml b/tests/integration/test_catboost_model_config_reload/config/catboost_lib.xml deleted file mode 100644 index 7aa06cc99ff..00000000000 --- a/tests/integration/test_catboost_model_config_reload/config/catboost_lib.xml +++ /dev/null @@ -1,3 +0,0 @@ - - /etc/clickhouse-server/model/libcatboostmodel.so - diff --git a/tests/integration/test_catboost_model_config_reload/config/models_config.xml b/tests/integration/test_catboost_model_config_reload/config/models_config.xml deleted file mode 100644 index 3cbf717bb67..00000000000 --- a/tests/integration/test_catboost_model_config_reload/config/models_config.xml +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tests/integration/test_catboost_model_config_reload/model/model_config.xml b/tests/integration/test_catboost_model_config_reload/model/model_config.xml deleted file mode 100644 index af9778097fa..00000000000 --- a/tests/integration/test_catboost_model_config_reload/model/model_config.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - catboost - model1 - /etc/clickhouse-server/model/model.bin - 0 - - diff --git a/tests/integration/test_catboost_model_config_reload/model/model_config2.xml b/tests/integration/test_catboost_model_config_reload/model/model_config2.xml deleted file mode 100644 index b81120ec900..00000000000 --- a/tests/integration/test_catboost_model_config_reload/model/model_config2.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - catboost - model2 - /etc/clickhouse-server/model/model.bin - 0 - - diff --git a/tests/integration/test_catboost_model_config_reload/test.py b/tests/integration/test_catboost_model_config_reload/test.py deleted file mode 100644 index c12c28e2338..00000000000 --- a/tests/integration/test_catboost_model_config_reload/test.py +++ /dev/null @@ -1,77 +0,0 @@ -import os -import sys -import time - -import pytest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance( - "node", - stay_alive=True, - main_configs=["config/models_config.xml", "config/catboost_lib.xml"], -) - - -def copy_file_to_container(local_path, dist_path, container_id): - os.system( - "docker cp {local} {cont_id}:{dist}".format( - local=local_path, cont_id=container_id, dist=dist_path - ) - ) - - -config = """ - /etc/clickhouse-server/model/{model_config} -""" - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - - copy_file_to_container( - os.path.join(SCRIPT_DIR, "model/."), - "/etc/clickhouse-server/model", - node.docker_id, - ) - node.restart_clickhouse() - - yield cluster - - finally: - cluster.shutdown() - - -def change_config(model_config): - node.replace_config( - "/etc/clickhouse-server/config.d/models_config.xml", - config.format(model_config=model_config), - ) - node.query("SYSTEM RELOAD CONFIG;") - - -def test(started_cluster): - if node.is_built_with_memory_sanitizer(): - pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - - # Set config with the path to the first model. - change_config("model_config.xml") - - node.query("SELECT modelEvaluate('model1', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);") - - # Change path to the second model in config. - change_config("model_config2.xml") - - # Check that the new model is loaded. - node.query("SELECT modelEvaluate('model2', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);") - - # Check that the old model was unloaded. - node.query_and_get_error( - "SELECT modelEvaluate('model1', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" - ) diff --git a/tests/integration/test_catboost_model_first_evaluate/config/models_config.xml b/tests/integration/test_catboost_model_first_evaluate/config/models_config.xml deleted file mode 100644 index 26f5c4d57f6..00000000000 --- a/tests/integration/test_catboost_model_first_evaluate/config/models_config.xml +++ /dev/null @@ -1,4 +0,0 @@ - - /etc/clickhouse-server/model/libcatboostmodel.so - /etc/clickhouse-server/model/model_config.xml - diff --git a/tests/integration/test_catboost_model_first_evaluate/model/libcatboostmodel.so b/tests/integration/test_catboost_model_first_evaluate/model/libcatboostmodel.so deleted file mode 100755 index 388d9f887b4..00000000000 Binary files a/tests/integration/test_catboost_model_first_evaluate/model/libcatboostmodel.so and /dev/null differ diff --git a/tests/integration/test_catboost_model_first_evaluate/model/model.bin b/tests/integration/test_catboost_model_first_evaluate/model/model.bin deleted file mode 100644 index 118e099d176..00000000000 Binary files a/tests/integration/test_catboost_model_first_evaluate/model/model.bin and /dev/null differ diff --git a/tests/integration/test_catboost_model_first_evaluate/model/model_config.xml b/tests/integration/test_catboost_model_first_evaluate/model/model_config.xml deleted file mode 100644 index 2c328167a94..00000000000 --- a/tests/integration/test_catboost_model_first_evaluate/model/model_config.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - catboost - titanic - /etc/clickhouse-server/model/model.bin - 0 - - diff --git a/tests/integration/test_catboost_model_first_evaluate/test.py b/tests/integration/test_catboost_model_first_evaluate/test.py deleted file mode 100644 index b15f481c0e9..00000000000 --- a/tests/integration/test_catboost_model_first_evaluate/test.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import sys -import time - -import pytest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance( - "node", stay_alive=True, main_configs=["config/models_config.xml"] -) - - -def copy_file_to_container(local_path, dist_path, container_id): - os.system( - "docker cp {local} {cont_id}:{dist}".format( - local=local_path, cont_id=container_id, dist=dist_path - ) - ) - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - - copy_file_to_container( - os.path.join(SCRIPT_DIR, "model/."), - "/etc/clickhouse-server/model", - node.docker_id, - ) - node.restart_clickhouse() - - yield cluster - - finally: - cluster.shutdown() - - -def test(started_cluster): - if node.is_built_with_memory_sanitizer(): - pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - - node.query("select modelEvaluate('titanic', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);") diff --git a/tests/integration/test_catboost_model_reload/config/catboost_lib.xml b/tests/integration/test_catboost_model_reload/config/catboost_lib.xml deleted file mode 100644 index 7aa06cc99ff..00000000000 --- a/tests/integration/test_catboost_model_reload/config/catboost_lib.xml +++ /dev/null @@ -1,3 +0,0 @@ - - /etc/clickhouse-server/model/libcatboostmodel.so - diff --git a/tests/integration/test_catboost_model_reload/config/models_config.xml b/tests/integration/test_catboost_model_reload/config/models_config.xml deleted file mode 100644 index 84378df0e8f..00000000000 --- a/tests/integration/test_catboost_model_reload/config/models_config.xml +++ /dev/null @@ -1,3 +0,0 @@ - - /etc/clickhouse-server/model/model_config.xml - diff --git a/tests/integration/test_catboost_model_reload/model/conjunction.cbm b/tests/integration/test_catboost_model_reload/model/conjunction.cbm deleted file mode 100644 index 7b75fb5f886..00000000000 Binary files a/tests/integration/test_catboost_model_reload/model/conjunction.cbm and /dev/null differ diff --git a/tests/integration/test_catboost_model_reload/model/disjunction.cbm b/tests/integration/test_catboost_model_reload/model/disjunction.cbm deleted file mode 100644 index 8145c24637f..00000000000 Binary files a/tests/integration/test_catboost_model_reload/model/disjunction.cbm and /dev/null differ diff --git a/tests/integration/test_catboost_model_reload/model/libcatboostmodel.so b/tests/integration/test_catboost_model_reload/model/libcatboostmodel.so deleted file mode 100755 index 388d9f887b4..00000000000 Binary files a/tests/integration/test_catboost_model_reload/model/libcatboostmodel.so and /dev/null differ diff --git a/tests/integration/test_catboost_model_reload/model/model_config.xml b/tests/integration/test_catboost_model_reload/model/model_config.xml deleted file mode 100644 index 7cbda165ce9..00000000000 --- a/tests/integration/test_catboost_model_reload/model/model_config.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - catboost - model - /etc/clickhouse-server/model/model.cbm - 0 - - diff --git a/tests/integration/test_catboost_model_reload/test.py b/tests/integration/test_catboost_model_reload/test.py deleted file mode 100644 index 3bf7ca18cdd..00000000000 --- a/tests/integration/test_catboost_model_reload/test.py +++ /dev/null @@ -1,132 +0,0 @@ -import os -import sys -import time - -import pytest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance( - "node", - stay_alive=True, - main_configs=["config/models_config.xml", "config/catboost_lib.xml"], -) - - -def copy_file_to_container(local_path, dist_path, container_id): - os.system( - "docker cp {local} {cont_id}:{dist}".format( - local=local_path, cont_id=container_id, dist=dist_path - ) - ) - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - - copy_file_to_container( - os.path.join(SCRIPT_DIR, "model/."), - "/etc/clickhouse-server/model", - node.docker_id, - ) - node.query("CREATE TABLE binary (x UInt64, y UInt64) ENGINE = TinyLog()") - node.query("INSERT INTO binary VALUES (1, 1), (1, 0), (0, 1), (0, 0)") - - node.restart_clickhouse() - - yield cluster - - finally: - cluster.shutdown() - - -def test_model_reload(started_cluster): - if node.is_built_with_memory_sanitizer(): - pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - - node.exec_in_container( - ["bash", "-c", "rm -f /etc/clickhouse-server/model/model.cbm"] - ) - node.exec_in_container( - [ - "bash", - "-c", - "ln /etc/clickhouse-server/model/conjunction.cbm /etc/clickhouse-server/model/model.cbm", - ] - ) - node.query("SYSTEM RELOAD MODEL model") - - result = node.query( - """ - WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability - SELECT if(probability > 0.5, 1, 0) FROM binary; - """ - ) - assert result == "1\n0\n0\n0\n" - - node.exec_in_container(["bash", "-c", "rm /etc/clickhouse-server/model/model.cbm"]) - node.exec_in_container( - [ - "bash", - "-c", - "ln /etc/clickhouse-server/model/disjunction.cbm /etc/clickhouse-server/model/model.cbm", - ] - ) - node.query("SYSTEM RELOAD MODEL model") - - result = node.query( - """ - WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability - SELECT if(probability > 0.5, 1, 0) FROM binary; - """ - ) - assert result == "1\n1\n1\n0\n" - - -def test_models_reload(started_cluster): - if node.is_built_with_memory_sanitizer(): - pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - - node.exec_in_container( - ["bash", "-c", "rm -f /etc/clickhouse-server/model/model.cbm"] - ) - node.exec_in_container( - [ - "bash", - "-c", - "ln /etc/clickhouse-server/model/conjunction.cbm /etc/clickhouse-server/model/model.cbm", - ] - ) - node.query("SYSTEM RELOAD MODELS") - - result = node.query( - """ - WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability - SELECT if(probability > 0.5, 1, 0) FROM binary; - """ - ) - assert result == "1\n0\n0\n0\n" - - node.exec_in_container(["bash", "-c", "rm /etc/clickhouse-server/model/model.cbm"]) - node.exec_in_container( - [ - "bash", - "-c", - "ln /etc/clickhouse-server/model/disjunction.cbm /etc/clickhouse-server/model/model.cbm", - ] - ) - node.query("SYSTEM RELOAD MODELS") - - result = node.query( - """ - WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability - SELECT if(probability > 0.5, 1, 0) FROM binary; - """ - ) - assert result == "1\n1\n1\n0\n" diff --git a/tests/integration/test_disabled_access_control_improvements/configs/config.d/disable_access_control_improvements.xml b/tests/integration/test_disabled_access_control_improvements/configs/config.d/disable_access_control_improvements.xml index 7969c638fd7..a335c7f8a1f 100644 --- a/tests/integration/test_disabled_access_control_improvements/configs/config.d/disable_access_control_improvements.xml +++ b/tests/integration/test_disabled_access_control_improvements/configs/config.d/disable_access_control_improvements.xml @@ -3,5 +3,6 @@ + diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index 273a00adffe..bcfc9d718a7 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -93,7 +93,7 @@ def test_alter(): instance.query("GRANT SELECT ON mydb.mytable TO u1") instance.query("GRANT SELECT ON mydb.* TO rx WITH GRANT OPTION") instance.query( - "ALTER SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY" + "ALTER SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 CONST" ) def check(): @@ -124,7 +124,7 @@ def test_alter(): ) assert ( instance.query("SHOW CREATE SETTINGS PROFILE s1") - == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n" + == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 CONST\n" ) assert ( instance.query("SHOW CREATE SETTINGS PROFILE s2") diff --git a/tests/integration/test_hive_query/test.py b/tests/integration/test_hive_query/test.py index b41c2dcd8a3..a498320ed5b 100644 --- a/tests/integration/test_hive_query/test.py +++ b/tests/integration/test_hive_query/test.py @@ -146,6 +146,8 @@ def test_orc_groupby(started_cluster): node = started_cluster.instances["h0_0_0"] result = node.query( """ + DROP TABLE IF EXISTS default.demo_orc; + CREATE TABLE default.demo_orc (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo_orc') PARTITION BY(day); SELECT day, count(*) FROM default.demo_orc group by day order by day """ ) @@ -329,6 +331,8 @@ def test_text_count(started_cluster): node = started_cluster.instances["h0_0_0"] result = node.query( """ + DROP TABLE IF EXISTS default.demo_orc; + CREATE TABLE default.demo_orc (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo_orc') PARTITION BY(day); SELECT day, count(*) FROM default.demo_orc group by day order by day SETTINGS format_csv_delimiter = '\x01' """ ) diff --git a/tests/integration/test_keeper_incorrect_config/test.py b/tests/integration/test_keeper_incorrect_config/test.py index e0a28b00b4f..cedb195a6e0 100644 --- a/tests/integration/test_keeper_incorrect_config/test.py +++ b/tests/integration/test_keeper_incorrect_config/test.py @@ -172,6 +172,37 @@ NORMAL_CONFIG = """ """ +JUST_WRONG_CONFIG = """ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + + 1 + node1 + 9234 + 2 + node2 + 9234 + 3 + node3 + 9234 + + + + +""" + def test_duplicate_endpoint(started_cluster): node1.stop_clickhouse() @@ -187,6 +218,7 @@ def test_duplicate_endpoint(started_cluster): assert_config_fails(DUPLICATE_ID_CONFIG) assert_config_fails(LOCALHOST_WITH_REMOTE) assert_config_fails(MULTIPLE_LOCAL_WITH_REMOTE) + assert_config_fails(JUST_WRONG_CONFIG) node1.replace_config( "/etc/clickhouse-server/config.d/enable_keeper1.xml", NORMAL_CONFIG diff --git a/tests/integration/test_s3_aws_sdk_is_total_garbage/__init__.py b/tests/integration/test_keeper_map/__init__.py similarity index 100% rename from tests/integration/test_s3_aws_sdk_is_total_garbage/__init__.py rename to tests/integration/test_keeper_map/__init__.py diff --git a/tests/integration/test_keeper_map/configs/enable_keeper_map.xml b/tests/integration/test_keeper_map/configs/enable_keeper_map.xml new file mode 100644 index 00000000000..b4cbb6a954b --- /dev/null +++ b/tests/integration/test_keeper_map/configs/enable_keeper_map.xml @@ -0,0 +1,3 @@ + + /test_keeper_map + diff --git a/tests/integration/test_keeper_map/test.py b/tests/integration/test_keeper_map/test.py new file mode 100644 index 00000000000..8f515077e8f --- /dev/null +++ b/tests/integration/test_keeper_map/test.py @@ -0,0 +1,179 @@ +import multiprocessing +import pytest +from time import sleep +import random +from itertools import count +from sys import stdout + +from multiprocessing import Pool + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry, assert_logs_contain +from helpers.network import PartitionManager + +test_recover_staled_replica_run = 1 + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node", + main_configs=["configs/enable_keeper_map.xml"], + with_zookeeper=True, + stay_alive=True, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def get_genuine_zk(): + return cluster.get_kazoo_client("zoo1") + + +def remove_children(client, path): + children = client.get_children(path) + + for child in children: + child_path = f"{path}/{child}" + remove_children(client, child_path) + client.delete(child_path) + + +def test_create_keeper_map(started_cluster): + node.query( + "CREATE TABLE test_keeper_map (key UInt64, value UInt64) ENGINE = KeeperMap('/test1') PRIMARY KEY(key);" + ) + zk_client = get_genuine_zk() + + def assert_children_size(path, expected_size): + assert len(zk_client.get_children(path)) == expected_size + + def assert_root_children_size(expected_size): + assert_children_size("/test_keeper_map/test1", expected_size) + + def assert_data_children_size(expected_size): + assert_children_size("/test_keeper_map/test1/data", expected_size) + + assert_root_children_size(2) + assert_data_children_size(0) + + node.query("INSERT INTO test_keeper_map VALUES (1, 11)") + assert_data_children_size(1) + + node.query( + "CREATE TABLE test_keeper_map_another (key UInt64, value UInt64) ENGINE = KeeperMap('/test1') PRIMARY KEY(key);" + ) + assert_root_children_size(2) + assert_data_children_size(1) + + node.query("INSERT INTO test_keeper_map_another VALUES (1, 11)") + assert_root_children_size(2) + assert_data_children_size(1) + + node.query("INSERT INTO test_keeper_map_another VALUES (2, 22)") + assert_root_children_size(2) + assert_data_children_size(2) + + node.query("DROP TABLE test_keeper_map SYNC") + assert_root_children_size(2) + assert_data_children_size(2) + + node.query("DROP TABLE test_keeper_map_another SYNC") + assert_root_children_size(0) + + zk_client.stop() + + +def create_drop_loop(index, stop_event): + table_name = f"test_keeper_map_{index}" + + for i in count(0, 1): + if stop_event.is_set(): + return + + node.query( + f"CREATE TABLE {table_name} (key UInt64, value UInt64) ENGINE = KeeperMap('/test') PRIMARY KEY(key);" + ) + node.query(f"INSERT INTO {table_name} VALUES ({index}, {i})") + result = node.query(f"SELECT value FROM {table_name} WHERE key = {index}") + assert result.strip() == str(i) + node.query(f"DROP TABLE {table_name} SYNC") + + +def test_create_drop_keeper_map_concurrent(started_cluster): + pool = Pool() + manager = multiprocessing.Manager() + stop_event = manager.Event() + results = [] + for i in range(multiprocessing.cpu_count()): + sleep(0.2) + results.append( + pool.apply_async( + create_drop_loop, + args=( + i, + stop_event, + ), + ) + ) + + sleep(60) + stop_event.set() + + for result in results: + result.get() + + pool.close() + + client = get_genuine_zk() + assert len(client.get_children("/test_keeper_map/test")) == 0 + client.stop() + + +def test_keeper_map_without_zk(started_cluster): + def assert_keeper_exception_after_partition(query): + with PartitionManager() as pm: + pm.drop_instance_zk_connections(node) + error = node.query_and_get_error(query) + assert "Coordination::Exception" in error + + assert_keeper_exception_after_partition( + "CREATE TABLE test_keeper_map (key UInt64, value UInt64) ENGINE = KeeperMap('/test1') PRIMARY KEY(key);" + ) + + node.query( + "CREATE TABLE test_keeper_map (key UInt64, value UInt64) ENGINE = KeeperMap('/test1') PRIMARY KEY(key);" + ) + + assert_keeper_exception_after_partition( + "INSERT INTO test_keeper_map VALUES (1, 11)" + ) + node.query("INSERT INTO test_keeper_map VALUES (1, 11)") + + assert_keeper_exception_after_partition("SELECT * FROM test_keeper_map") + node.query("SELECT * FROM test_keeper_map") + + with PartitionManager() as pm: + pm.drop_instance_zk_connections(node) + node.restart_clickhouse(60) + error = node.query_and_get_error("SELECT * FROM test_keeper_map") + assert "Failed to activate table because of connection issues" in error + + node.query("SELECT * FROM test_keeper_map") + + client = get_genuine_zk() + remove_children(client, "/test_keeper_map/test1") + node.restart_clickhouse(60) + error = node.query_and_get_error("SELECT * FROM test_keeper_map") + assert "Failed to activate table because of invalid metadata in ZooKeeper" in error + + node.query("DETACH TABLE test_keeper_map") + + client.stop() diff --git a/tests/integration/test_keeper_session/configs/keeper_config.xml b/tests/integration/test_keeper_session/configs/keeper_config1.xml similarity index 67% rename from tests/integration/test_keeper_session/configs/keeper_config.xml rename to tests/integration/test_keeper_session/configs/keeper_config1.xml index ed0bb52bd51..fd308fe8a2f 100644 --- a/tests/integration/test_keeper_session/configs/keeper_config.xml +++ b/tests/integration/test_keeper_session/configs/keeper_config1.xml @@ -1,4 +1,4 @@ - + 9181 1 @@ -19,9 +19,19 @@ 1 node1 9234 - true - 3 + + + 2 + node2 + 9234 + true + + + 3 + node3 + 9234 + true - + diff --git a/tests/integration/test_keeper_session/configs/keeper_config2.xml b/tests/integration/test_keeper_session/configs/keeper_config2.xml new file mode 100644 index 00000000000..ad558fbccad --- /dev/null +++ b/tests/integration/test_keeper_session/configs/keeper_config2.xml @@ -0,0 +1,37 @@ + + + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + * + + + 5000 + 10000 + 5000 + 75 + trace + + + + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + true + + + 3 + node3 + 9234 + true + + + + diff --git a/tests/integration/test_keeper_session/configs/keeper_config3.xml b/tests/integration/test_keeper_session/configs/keeper_config3.xml new file mode 100644 index 00000000000..2a21f959816 --- /dev/null +++ b/tests/integration/test_keeper_session/configs/keeper_config3.xml @@ -0,0 +1,37 @@ + + + 9181 + 3 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + * + + + 5000 + 10000 + 5000 + 75 + trace + + + + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + true + + + 3 + node3 + 9234 + true + + + + diff --git a/tests/integration/test_keeper_session/test.py b/tests/integration/test_keeper_session/test.py index 30db4d9548c..4b3aa7e3fdf 100644 --- a/tests/integration/test_keeper_session/test.py +++ b/tests/integration/test_keeper_session/test.py @@ -10,7 +10,15 @@ from kazoo.client import KazooClient cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( - "node1", main_configs=["configs/keeper_config.xml"], stay_alive=True + "node1", main_configs=["configs/keeper_config1.xml"], stay_alive=True +) + +node2 = cluster.add_instance( + "node2", main_configs=["configs/keeper_config2.xml"], stay_alive=True +) + +node3 = cluster.add_instance( + "node3", main_configs=["configs/keeper_config3.xml"], stay_alive=True ) bool_struct = struct.Struct("B") @@ -61,7 +69,7 @@ def wait_node(node): def wait_nodes(): - for n in [node1]: + for n in [node1, node2, node3]: wait_node(n) @@ -165,3 +173,21 @@ def test_session_timeout(started_cluster): negotiated_timeout, _ = handshake(node1.name, session_timeout=20000, session_id=0) assert negotiated_timeout == 10000 + + +def test_session_close_shutdown(started_cluster): + wait_nodes() + + node1_zk = get_fake_zk(node1.name) + node2_zk = get_fake_zk(node2.name) + + eph_node = "/test_node" + node2_zk.create(eph_node, ephemeral=True) + assert node1_zk.exists(eph_node) != None + + # shutdown while session is active + node2.stop_clickhouse() + + assert node1_zk.exists(eph_node) == None + + node2.start_clickhouse() diff --git a/tests/integration/test_catboost_model_first_evaluate/__init__.py b/tests/integration/test_mask_queries_in_logs/__init__.py similarity index 100% rename from tests/integration/test_catboost_model_first_evaluate/__init__.py rename to tests/integration/test_mask_queries_in_logs/__init__.py diff --git a/tests/integration/test_mask_queries_in_logs/test.py b/tests/integration/test_mask_queries_in_logs/test.py new file mode 100644 index 00000000000..9ab13f41c9e --- /dev/null +++ b/tests/integration/test_mask_queries_in_logs/test.py @@ -0,0 +1,99 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node") + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +# Passwords in CREATE/ALTER queries must be hidden in logs. +def test_create_alter_user(): + node.query("CREATE USER u1 IDENTIFIED BY 'qwe123' SETTINGS custom_a = 'a'") + node.query("ALTER USER u1 IDENTIFIED BY '123qwe' SETTINGS custom_b = 'b'") + node.query( + "CREATE USER u2 IDENTIFIED WITH plaintext_password BY 'plainpasswd' SETTINGS custom_c = 'c'" + ) + + assert ( + node.query("SHOW CREATE USER u1") + == "CREATE USER u1 IDENTIFIED WITH sha256_password SETTINGS custom_b = \\'b\\'\n" + ) + assert ( + node.query("SHOW CREATE USER u2") + == "CREATE USER u2 IDENTIFIED WITH plaintext_password SETTINGS custom_c = \\'c\\'\n" + ) + + node.query("SYSTEM FLUSH LOGS") + + assert node.contains_in_log("CREATE USER u1") + assert node.contains_in_log("ALTER USER u1") + assert node.contains_in_log("CREATE USER u2") + assert not node.contains_in_log("qwe123") + assert not node.contains_in_log("123qwe") + assert not node.contains_in_log("plainpasswd") + assert not node.contains_in_log("IDENTIFIED WITH sha256_password BY") + assert not node.contains_in_log("IDENTIFIED WITH sha256_hash BY") + assert not node.contains_in_log("IDENTIFIED WITH plaintext_password BY") + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'CREATE USER u1%IDENTIFIED WITH sha256_password%'" + ).strip() + ) + >= 1 + ) + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'CREATE USER u1%IDENTIFIED WITH sha256_password BY%'" + ).strip() + ) + == 0 + ) + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'ALTER USER u1%IDENTIFIED WITH sha256_password%'" + ).strip() + ) + >= 1 + ) + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'ALTER USER u1%IDENTIFIED WITH sha256_password BY%'" + ).strip() + ) + == 0 + ) + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'CREATE USER u2%IDENTIFIED WITH plaintext_password%'" + ).strip() + ) + >= 1 + ) + + assert ( + int( + node.query( + "SELECT COUNT() FROM system.query_log WHERE query LIKE 'CREATE USER u2%IDENTIFIED WITH plaintext_password BY%'" + ).strip() + ) + == 0 + ) diff --git a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml index 3ee49744a61..f3505f53339 100644 --- a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml @@ -38,6 +38,20 @@ /jbod1/ 1000000000 + + s3 + http://minio1:9001/root/data/ + minio + minio123 + 33554432 + + + cache + s3_r + /s3_cache_r/ + 1000000000 + 1 + @@ -78,6 +92,13 @@ + + +
+ s3_cache_r +
+ + diff --git a/tests/integration/test_merge_tree_s3/configs/config.xml b/tests/integration/test_merge_tree_s3/configs/config.xml index feb537ebbce..8055a5dd0a5 100644 --- a/tests/integration/test_merge_tree_s3/configs/config.xml +++ b/tests/integration/test_merge_tree_s3/configs/config.xml @@ -1,18 +1,3 @@ - 9000 - 127.0.0.1 - - - - true - none - - AcceptCertificateHandler - - - - - 500 - ./clickhouse/ - users.xml + true diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index 4276125c347..912457afaa1 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -6,7 +6,6 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.utility import generate_values, replace_config, SafeThread - SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -17,6 +16,7 @@ def cluster(): cluster.add_instance( "node", main_configs=[ + "configs/config.xml", "configs/config.d/storage_conf.xml", "configs/config.d/bg_processing_pool_conf.xml", ], @@ -35,6 +35,7 @@ def cluster(): "/jbod1:size=2M", ], ) + logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -541,6 +542,8 @@ def test_freeze_unfreeze(cluster, node_name): # Unfreeze all partitions from backup2. node.query("ALTER TABLE s3_test UNFREEZE WITH NAME 'backup2'") + wait_for_delete_s3_objects(cluster, FILES_OVERHEAD) + # Data should be removed from S3. assert ( len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) @@ -574,6 +577,8 @@ def test_freeze_system_unfreeze(cluster, node_name): # Unfreeze all data from backup3. node.query("SYSTEM UNFREEZE WITH NAME 'backup3'") + wait_for_delete_s3_objects(cluster, FILES_OVERHEAD) + # Data should be removed from S3. assert ( len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) @@ -749,3 +754,79 @@ def test_store_cleanup_disk_s3(cluster, node_name): "CREATE TABLE s3_test UUID '00000000-1000-4000-8000-000000000001' (n UInt64) Engine=MergeTree() ORDER BY n SETTINGS storage_policy='s3';" ) node.query("INSERT INTO s3_test SELECT 1") + + +@pytest.mark.parametrize("node_name", ["node"]) +def test_cache_setting_compatibility(cluster, node_name): + node = cluster.instances[node_name] + + node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + + node.query( + "CREATE TABLE s3_test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_r';" + ) + node.query( + "INSERT INTO s3_test SELECT * FROM generateRandom('key UInt32, value String') LIMIT 500" + ) + + result = node.query("SYSTEM DROP FILESYSTEM CACHE") + + result = node.query( + "SELECT count() FROM system.filesystem_cache WHERE cache_path LIKE '%persistent'" + ) + assert int(result) == 0 + + node.query("SELECT * FROM s3_test") + + result = node.query( + "SELECT count() FROM system.filesystem_cache WHERE cache_path LIKE '%persistent'" + ) + assert int(result) > 0 + + config_path = os.path.join( + SCRIPT_DIR, + f"./{cluster.instances_dir_name}/node/configs/config.d/storage_conf.xml", + ) + + replace_config( + config_path, + "1", + "0", + ) + + result = node.query("DESCRIBE CACHE 's3_cache_r'") + assert result.strip().endswith("1") + + node.restart_clickhouse() + + result = node.query("DESCRIBE CACHE 's3_cache_r'") + assert result.strip().endswith("0") + + result = node.query( + "SELECT count() FROM system.filesystem_cache WHERE cache_path LIKE '%persistent'" + ) + assert int(result) > 0 + + node.query("SELECT * FROM s3_test FORMAT Null") + + assert not node.contains_in_log("No such file or directory: Cache info:") + + replace_config( + config_path, + "0", + "1", + ) + + result = node.query( + "SELECT count() FROM system.filesystem_cache WHERE cache_path LIKE '%persistent'" + ) + assert int(result) > 0 + + node.restart_clickhouse() + + result = node.query("DESCRIBE CACHE 's3_cache_r'") + assert result.strip().endswith("1") + + node.query("SELECT * FROM s3_test FORMAT Null") + + assert not node.contains_in_log("No such file or directory: Cache info:") diff --git a/tests/integration/test_profile_events_s3/test.py b/tests/integration/test_profile_events_s3/test.py index 18f1c5ee9ad..a7e22ed5516 100644 --- a/tests/integration/test_profile_events_s3/test.py +++ b/tests/integration/test_profile_events_s3/test.py @@ -56,6 +56,28 @@ init_list = { "DiskS3WriteRequestsErrorsTotal": 0, "DiskS3WriteRequestsErrors503": 0, "DiskS3WriteRequestsRedirects": 0, + "S3DeleteObjects": 0, + "S3CopyObject": 0, + "S3ListObjects": 0, + "S3HeadObject": 0, + "S3CreateMultipartUpload": 0, + "S3UploadPartCopy": 0, + "S3UploadPart": 0, + "S3AbortMultipartUpload": 0, + "S3CompleteMultipartUpload": 0, + "S3PutObject": 0, + "S3GetObject": 0, + "DiskS3DeleteObjects": 0, + "DiskS3CopyObject": 0, + "DiskS3ListObjects": 0, + "DiskS3HeadObject": 0, + "DiskS3CreateMultipartUpload": 0, + "DiskS3UploadPartCopy": 0, + "DiskS3UploadPart": 0, + "DiskS3AbortMultipartUpload": 0, + "DiskS3CompleteMultipartUpload": 0, + "DiskS3PutObject": 0, + "DiskS3GetObject": 0, } diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 0cf237d57f3..de5433d5beb 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -354,6 +354,42 @@ def test_alter_drop_detached_part(started_cluster, engine): dummy_node.query("DROP DATABASE testdb SYNC") +@pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +def test_alter_drop_partition(started_cluster, engine): + main_node.query( + "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/clickhouse/databases/test_alter_drop_partition', 'shard1', 'replica1');" + ) + dummy_node.query( + "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/clickhouse/databases/test_alter_drop_partition', 'shard1', 'replica2');" + ) + snapshotting_node.query( + "CREATE DATABASE alter_drop_partition ENGINE = Replicated('/clickhouse/databases/test_alter_drop_partition', 'shard2', 'replica1');" + ) + + table = f"alter_drop_partition.alter_drop_{engine}" + main_node.query( + f"CREATE TABLE {table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + ) + main_node.query(f"INSERT INTO {table} VALUES (123)") + if engine == "MergeTree": + dummy_node.query(f"INSERT INTO {table} VALUES (456)") + snapshotting_node.query(f"INSERT INTO {table} VALUES (789)") + main_node.query( + f"ALTER TABLE {table} ON CLUSTER alter_drop_partition DROP PARTITION ID 'all'", + settings={"replication_alter_partitions_sync": 2}, + ) + assert ( + main_node.query( + f"SELECT CounterID FROM clusterAllReplicas('alter_drop_partition', {table})" + ) + == "" + ) + assert dummy_node.query(f"SELECT CounterID FROM {table}") == "" + main_node.query("DROP DATABASE alter_drop_partition") + dummy_node.query("DROP DATABASE alter_drop_partition") + snapshotting_node.query("DROP DATABASE alter_drop_partition") + + def test_alter_fetch(started_cluster): main_node.query( "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" diff --git a/tests/integration/test_replicated_users/configs/zookeeper.xml b/tests/integration/test_replicated_users/configs/zookeeper.xml new file mode 100644 index 00000000000..3176f4959ee --- /dev/null +++ b/tests/integration/test_replicated_users/configs/zookeeper.xml @@ -0,0 +1,17 @@ + + + + zoo1 + 2181 + + + zoo2 + 2181 + + + zoo3 + 2181 + + 20000 + + diff --git a/tests/integration/test_replicated_users/test.py b/tests/integration/test_replicated_users/test.py index 56383f0d2df..4fb2408c46f 100644 --- a/tests/integration/test_replicated_users/test.py +++ b/tests/integration/test_replicated_users/test.py @@ -1,15 +1,24 @@ import pytest +import time from dataclasses import dataclass from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry, TSV -cluster = ClickHouseCluster(__file__) +cluster = ClickHouseCluster(__file__, zookeeper_config_path="configs/zookeeper.xml") node1 = cluster.add_instance( - "node1", main_configs=["configs/config.xml"], with_zookeeper=True, stay_alive=True + "node1", + main_configs=["configs/config.xml"], + with_zookeeper=True, + stay_alive=True, ) + node2 = cluster.add_instance( - "node2", main_configs=["configs/config.xml"], with_zookeeper=True, stay_alive=True + "node2", + main_configs=["configs/config.xml"], + with_zookeeper=True, + stay_alive=True, ) all_nodes = [node1, node2] @@ -88,3 +97,113 @@ def test_rename_replicated(started_cluster, entity): f"ALTER {entity.keyword} {entity.name} {entity.options} RENAME TO {entity.name}2" ) node1.query(f"DROP {entity.keyword} {entity.name}2 {entity.options}") + + +# ReplicatedAccessStorage must be able to continue working after reloading ZooKeeper. +def test_reload_zookeeper(started_cluster): + def wait_zookeeper_node_to_start(zk_nodes, timeout=60): + start = time.time() + while time.time() - start < timeout: + try: + for instance in zk_nodes: + conn = cluster.get_kazoo_client(instance) + conn.get_children("/") + print("All instances of ZooKeeper started") + return + except Exception as ex: + print(("Can't connect to ZooKeeper " + str(ex))) + time.sleep(0.5) + + def replace_zookeeper_config(new_config): + node1.replace_config("/etc/clickhouse-server/conf.d/zookeeper.xml", new_config) + node2.replace_config("/etc/clickhouse-server/conf.d/zookeeper.xml", new_config) + node1.query("SYSTEM RELOAD CONFIG") + node2.query("SYSTEM RELOAD CONFIG") + + def get_active_zk_connections(): + return str( + node1.exec_in_container( + [ + "bash", + "-c", + "lsof -a -i4 -i6 -itcp -w | grep 2181 | grep ESTABLISHED | wc -l", + ], + privileged=True, + user="root", + ) + ).strip() + + node1.query("CREATE USER u1") + assert_eq_with_retry( + node2, "SELECT name FROM system.users WHERE name ='u1'", "u1\n" + ) + + ## remove zoo2, zoo3 from configs + replace_zookeeper_config( + """ + + + + zoo1 + 2181 + + 2000 + + +""" + ) + + ## config reloads, but can still work + node1.query("CREATE USER u2") + assert_eq_with_retry( + node2, + "SELECT name FROM system.users WHERE name IN ['u1', 'u2'] ORDER BY name", + TSV(["u1", "u2"]), + ) + + ## stop all zookeepers, users will be readonly + cluster.stop_zookeeper_nodes(["zoo1", "zoo2", "zoo3"]) + assert node2.query( + "SELECT name FROM system.users WHERE name IN ['u1', 'u2'] ORDER BY name" + ) == TSV(["u1", "u2"]) + assert "ZooKeeper" in node1.query_and_get_error("CREATE USER u3") + + ## start zoo2, zoo3, users will be readonly too, because it only connect to zoo1 + cluster.start_zookeeper_nodes(["zoo2", "zoo3"]) + wait_zookeeper_node_to_start(["zoo2", "zoo3"]) + assert node2.query( + "SELECT name FROM system.users WHERE name IN ['u1', 'u2'] ORDER BY name" + ) == TSV(["u1", "u2"]) + assert "ZooKeeper" in node1.query_and_get_error("CREATE USER u3") + + ## set config to zoo2, server will be normal + replace_zookeeper_config( + """ + + + + zoo2 + 2181 + + 2000 + + +""" + ) + + active_zk_connections = get_active_zk_connections() + assert ( + active_zk_connections == "1" + ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections) + + node1.query("CREATE USER u3") + assert_eq_with_retry( + node2, + "SELECT name FROM system.users WHERE name IN ['u1', 'u2', 'u3'] ORDER BY name", + TSV(["u1", "u2", "u3"]), + ) + + active_zk_connections = get_active_zk_connections() + assert ( + active_zk_connections == "1" + ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections) diff --git a/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/__init__.py b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_s3_aws_sdk_is_total_garbage/configs/storage_conf.xml b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/configs/storage_conf.xml similarity index 100% rename from tests/integration/test_s3_aws_sdk_is_total_garbage/configs/storage_conf.xml rename to tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/configs/storage_conf.xml diff --git a/tests/integration/test_s3_aws_sdk_is_total_garbage/configs/upload_min_size.xml b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/configs/upload_min_size.xml similarity index 100% rename from tests/integration/test_s3_aws_sdk_is_total_garbage/configs/upload_min_size.xml rename to tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/configs/upload_min_size.xml diff --git a/tests/integration/test_s3_aws_sdk_is_total_garbage/s3_endpoint/endpoint.py b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py similarity index 100% rename from tests/integration/test_s3_aws_sdk_is_total_garbage/s3_endpoint/endpoint.py rename to tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py diff --git a/tests/integration/test_s3_aws_sdk_is_total_garbage/test.py b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/test.py similarity index 100% rename from tests/integration/test_s3_aws_sdk_is_total_garbage/test.py rename to tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/test.py diff --git a/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml b/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml index bc5273036cb..f7d9efc2cae 100644 --- a/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml +++ b/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml @@ -1,5 +1,5 @@ - + true diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test.py index 90a6c684de7..83c0827f891 100644 --- a/tests/integration/test_send_crash_reports/test.py +++ b/tests/integration/test_send_crash_reports/test.py @@ -36,8 +36,10 @@ def started_node(): def test_send_segfault(started_node): + # NOTE: another option is to increase waiting time. if ( started_node.is_built_with_thread_sanitizer() + or started_node.is_built_with_address_sanitizer() or started_node.is_built_with_memory_sanitizer() ): pytest.skip("doesn't fit in timeouts for stacktrace generation") diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 13af1cf650d..3358315cca7 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -216,7 +216,18 @@ def test_settings_from_granted_role(): "\\N", "\\N", ], - ["xyz", "\\N", "\\N", 1, "max_ast_depth", 2000, "\\N", "\\N", "\\N", "\\N"], + [ + "xyz", + "\\N", + "\\N", + 1, + "max_ast_depth", + 2000, + "\\N", + "\\N", + "\\N", + "\\N", + ], ] assert system_settings_profile_elements(role_name="worker") == [ ["\\N", "\\N", "worker", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] @@ -288,12 +299,12 @@ def test_settings_from_granted_role(): def test_inheritance(): instance.query( - "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY" + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 CONST" ) instance.query("CREATE SETTINGS PROFILE alpha SETTINGS PROFILE xyz TO robin") assert ( instance.query("SHOW CREATE SETTINGS PROFILE xyz") - == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY\n" + == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 CONST\n" ) assert ( instance.query("SHOW CREATE SETTINGS PROFILE alpha") @@ -315,7 +326,18 @@ def test_inheritance(): ["xyz", "local directory", 1, 0, "[]", "[]"] ] assert system_settings_profile_elements(profile_name="xyz") == [ - ["xyz", "\\N", "\\N", 0, "max_memory_usage", 100000002, "\\N", "\\N", 1, "\\N"] + [ + "xyz", + "\\N", + "\\N", + 0, + "max_memory_usage", + 100000002, + "\\N", + "\\N", + "CONST", + "\\N", + ] ] assert system_settings_profile("alpha") == [ ["alpha", "local directory", 1, 0, "['robin']", "[]"] @@ -366,6 +388,66 @@ def test_alter_and_drop(): instance.query("SET max_memory_usage = 120000000", user="robin") +def test_changeable_in_readonly(): + instance.query( + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000003 MIN 90000000 MAX 110000000 CHANGEABLE_IN_READONLY SETTINGS readonly = 1 TO robin" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "100000003\n" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'readonly'", + user="robin", + ) + == "1\n" + ) + assert ( + "Setting max_memory_usage shouldn't be less than 90000000" + in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + ) + assert ( + "Setting max_memory_usage shouldn't be greater than 110000000" + in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin" + ) + ) + + assert system_settings_profile_elements(profile_name="xyz") == [ + [ + "xyz", + "\\N", + "\\N", + 0, + "max_memory_usage", + 100000003, + 90000000, + 110000000, + "CHANGEABLE_IN_READONLY", + "\\N", + ], + [ + "xyz", + "\\N", + "\\N", + 1, + "readonly", + 1, + "\\N", + "\\N", + "\\N", + "\\N", + ], + ] + + instance.query("SET max_memory_usage = 90000000", user="robin") + instance.query("SET max_memory_usage = 110000000", user="robin") + + def test_show_profiles(): instance.query("CREATE SETTINGS PROFILE xyz") assert instance.query("SHOW SETTINGS PROFILES") == "default\nreadonly\nxyz\n" diff --git a/tests/performance/avg_weighted.xml b/tests/performance/avg_weighted.xml index df992ad682a..5aa89b08c35 100644 --- a/tests/performance/avg_weighted.xml +++ b/tests/performance/avg_weighted.xml @@ -32,5 +32,21 @@ SELECT avgWeighted(num_u, num) FROM perf_avg FORMAT Null SELECT avgWeighted(num_u, num_u) FROM perf_avg FORMAT Null + SELECT avgWeighted(num_f, num_f) FROM perf_avg FORMAT Null + SELECT avgWeighted(toNullable(num_f), num_f) FROM perf_avg FORMAT Null + SELECT avgWeighted(num_f, toNullable(num_f)) FROM perf_avg FORMAT Null + SELECT avgWeighted(toNullable(num_f), toNullable(num_f)) FROM perf_avg FORMAT Null + + SELECT avgWeightedIf(num_f, num_f, num % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(toNullable(num_f), num_f, num % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(num_f, toNullable(num_f), num % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(toNullable(num_f), toNullable(num_f), num % 10) FROM perf_avg FORMAT Null + + SELECT avgWeightedIf(num_f, num_f, toNullable(num) % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(toNullable(num_f), num_f, toNullable(num) % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(num_f, toNullable(num_f), toNullable(num) % 10) FROM perf_avg FORMAT Null + SELECT avgWeightedIf(toNullable(num_f), toNullable(num_f), toNullable(num) % 10) FROM perf_avg FORMAT Null + + DROP TABLE IF EXISTS perf_avg diff --git a/tests/performance/lower_upper_utf8.xml b/tests/performance/lower_upper_utf8.xml new file mode 100644 index 00000000000..cc0c3af983b --- /dev/null +++ b/tests/performance/lower_upper_utf8.xml @@ -0,0 +1,4 @@ + + SELECT lowerUTF8(SearchPhrase) FROM hits_100m_single FORMAT Null + SELECT upperUTF8(SearchPhrase) FROM hits_100m_single FORMAT Null + diff --git a/tests/performance/uniq_stored.xml b/tests/performance/uniq_stored.xml new file mode 100644 index 00000000000..15d039978c7 --- /dev/null +++ b/tests/performance/uniq_stored.xml @@ -0,0 +1,58 @@ + + + create table matview_1 + ( + a String, + b_count AggregateFunction(uniq, UInt64) + ) Engine=MergeTree partition by tuple() + ORDER by a + SETTINGS index_granularity = 1024; + + + + create table matview_10000 + ( + a String, + b_count AggregateFunction(uniq, String) + ) Engine=MergeTree partition by tuple() + ORDER by a + SETTINGS index_granularity = 1024; + + + + DROP TABLE IF EXISTS matview_1 + DROP TABLE IF EXISTS matview_10000 + + + INSERT INTO matview_10000 + SELECT a, uniqState(b) b_count + FROM + ( + SELECT toString(intDiv(number, 20000)) a, toString(number % 10000) b + FROM numbers_mt(20000000) + ) + GROUP BY a + SETTINGS max_insert_threads=8; + + OPTIMIZE TABLE matview_10000 FINAL + + + INSERT INTO matview_1 + SELECT '1', uniqState(number) b_count + FROM + ( + SELECT * + FROM numbers_mt(2000000) + ) + GROUP BY number + SETTINGS max_insert_threads=8; + + OPTIMIZE TABLE matview_1 FINAL + + + select a, uniqMerge(b_count) as b_count from matview_10000 prewhere a='55' group by a FORMAT Null SETTINGS max_threads=1; + select uniqMerge(b_count) as b_count from matview_10000 FORMAT Null SETTINGS max_threads=1; + + + select uniqMerge(b_count) as b_count FROM matview_1 FORMAT Null SETTINGS max_threads=1; + diff --git a/tests/queries/0_stateless/00118_storage_join.reference b/tests/queries/0_stateless/00118_storage_join.reference index 56920b290e6..7fd76339287 100644 --- a/tests/queries/0_stateless/00118_storage_join.reference +++ b/tests/queries/0_stateless/00118_storage_join.reference @@ -39,9 +39,9 @@ 8 0 8 9 0 9 0 3 -3 9 -2 21 def 1 12 abc +2 21 def +3 9 0 45 0 0 diff --git a/tests/queries/0_stateless/00118_storage_join.sql b/tests/queries/0_stateless/00118_storage_join.sql index a855dcd56d8..52b0df4d796 100644 --- a/tests/queries/0_stateless/00118_storage_join.sql +++ b/tests/queries/0_stateless/00118_storage_join.sql @@ -3,19 +3,19 @@ DROP TABLE IF EXISTS t2; CREATE TABLE t2 (k UInt64, s String) ENGINE = Join(ANY, LEFT, k); INSERT INTO t2 VALUES (1, 'abc'), (2, 'def'); -SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k ORDER BY k; INSERT INTO t2 VALUES (6, 'ghi'); -SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k ORDER BY k; -SELECT k, js1.s, t2.s FROM (SELECT number AS k, number as s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; -SELECT k, t2.k, js1.s, t2.s FROM (SELECT number AS k, number as s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT k, js1.s, t2.s FROM (SELECT number AS k, number as s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k ORDER BY k; +SELECT k, t2.k, js1.s, t2.s FROM (SELECT number AS k, number as s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k ORDER BY k; -SELECT k, js1.s, t2.s FROM (SELECT toUInt64(number / 3) AS k, sum(number) as s FROM numbers(10) GROUP BY toUInt64(number / 3) WITH TOTALS) js1 ANY LEFT JOIN t2 USING k; +SELECT k, js1.s, t2.s FROM (SELECT toUInt64(number / 3) AS k, sum(number) as s FROM numbers(10) GROUP BY toUInt64(number / 3) WITH TOTALS) js1 ANY LEFT JOIN t2 USING k ORDER BY k; -SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k; -SELECT k, t2.k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k; +SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k ORDER BY k; +SELECT k, t2.k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k ORDER BY k; -SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k OR js1.s == t2.k; -- { serverError 48 } +SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k OR js1.s == t2.k ORDER BY k; -- { serverError 48 } DROP TABLE t2; diff --git a/tests/queries/0_stateless/00632_get_sample_block_cache.sql b/tests/queries/0_stateless/00632_get_sample_block_cache.sql index cbb89387dd1..c54ca0b084e 100644 --- a/tests/queries/0_stateless/00632_get_sample_block_cache.sql +++ b/tests/queries/0_stateless/00632_get_sample_block_cache.sql @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-s3-storage, no-asan SET joined_subquery_requires_alias = 0; diff --git a/tests/queries/0_stateless/00646_url_engine.python b/tests/queries/0_stateless/00646_url_engine.python index d1836817867..c955eaff643 100644 --- a/tests/queries/0_stateless/00646_url_engine.python +++ b/tests/queries/0_stateless/00646_url_engine.python @@ -194,7 +194,7 @@ def main(): select_requests = { "select distinct numuint from {tbl} order by numuint": '\n'.join([str(i) for i in range(11)]), "select count(*) from {tbl}": '12', - 'select double, count(*) from {tbl} group by double': "7.7\t2\n9.9\t10" + 'select double, count(*) from {tbl} group by double order by double': "7.7\t2\n9.9\t10" } t, httpd = start_server() diff --git a/tests/queries/0_stateless/00862_decimal_in.sql b/tests/queries/0_stateless/00862_decimal_in.sql index 2e24ea6d6ac..b5c058119a2 100644 --- a/tests/queries/0_stateless/00862_decimal_in.sql +++ b/tests/queries/0_stateless/00862_decimal_in.sql @@ -1,18 +1,17 @@ DROP TABLE IF EXISTS temp; CREATE TABLE temp ( - x Decimal(38,2), - y Nullable(Decimal(38,2)) + x Decimal(38, 2), + y Nullable(Decimal(38, 2)) ) ENGINE = Memory; INSERT INTO temp VALUES (32, 32), (64, 64), (128, 128); -SELECT * FROM temp WHERE x IN (toDecimal128(128, 2)); -SELECT * FROM temp WHERE y IN (toDecimal128(128, 2)); - SELECT * FROM temp WHERE x IN (toDecimal128(128, 1)); +SELECT * FROM temp WHERE x IN (toDecimal128(128, 2)); SELECT * FROM temp WHERE x IN (toDecimal128(128, 3)); SELECT * FROM temp WHERE y IN (toDecimal128(128, 1)); +SELECT * FROM temp WHERE y IN (toDecimal128(128, 2)); SELECT * FROM temp WHERE y IN (toDecimal128(128, 3)); SELECT * FROM temp WHERE x IN (toDecimal32(32, 1)); @@ -29,4 +28,7 @@ SELECT * FROM temp WHERE y IN (toDecimal64(64, 1)); SELECT * FROM temp WHERE y IN (toDecimal64(64, 2)); SELECT * FROM temp WHERE y IN (toDecimal64(64, 3)); +SELECT * FROM temp WHERE x IN (toDecimal256(256, 1)); -- { serverError 53 } +SELECT * FROM temp WHERE y IN (toDecimal256(256, 1)); -- { serverError 53 } + DROP TABLE IF EXISTS temp; diff --git a/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql b/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql index 5b763d2d853..555e7a98380 100644 --- a/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql +++ b/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql @@ -2,7 +2,7 @@ SET prefer_localhost_replica = 1; -SELECT count() FROM remote('127.0.0.1,localhos', system.one); -- { serverError 279 } +SELECT count() FROM remote('127.0.0.1,localhos', system.one); -- { serverError 198 } SELECT count() FROM remote('127.0.0.1|localhos', system.one); -- Clear cache to avoid future errors in the logs diff --git a/tests/queries/0_stateless/01050_engine_join_crash.reference b/tests/queries/0_stateless/01050_engine_join_crash.reference index f1a4d615cc0..00a2eafc310 100644 --- a/tests/queries/0_stateless/01050_engine_join_crash.reference +++ b/tests/queries/0_stateless/01050_engine_join_crash.reference @@ -4,9 +4,9 @@ 1 1 2 2 3 3 -3 3 -2 2 1 1 +2 2 +3 3 - 1 52.5 ONE - diff --git a/tests/queries/0_stateless/01050_engine_join_crash.sql b/tests/queries/0_stateless/01050_engine_join_crash.sql index 3dd4bd2b798..7a71550bb1d 100644 --- a/tests/queries/0_stateless/01050_engine_join_crash.sql +++ b/tests/queries/0_stateless/01050_engine_join_crash.sql @@ -8,9 +8,9 @@ CREATE TABLE testJoinTable (number UInt64, data String) ENGINE = Join(ANY, INNER INSERT INTO testJoinTable VALUES (1, '1'), (2, '2'), (3, '3'); SELECT * FROM (SELECT * FROM numbers(10)) js1 INNER JOIN testJoinTable USING number; -- { serverError 264 } -SELECT * FROM (SELECT * FROM numbers(10)) js1 INNER JOIN (SELECT * FROM testJoinTable) js2 USING number; -SELECT * FROM (SELECT * FROM numbers(10)) js1 ANY INNER JOIN testJoinTable USING number; -SELECT * FROM testJoinTable; +SELECT * FROM (SELECT * FROM numbers(10)) js1 INNER JOIN (SELECT * FROM testJoinTable) js2 USING number ORDER BY number; +SELECT * FROM (SELECT * FROM numbers(10)) js1 ANY INNER JOIN testJoinTable USING number ORDER BY number; +SELECT * FROM testJoinTable ORDER BY number; DROP TABLE testJoinTable; @@ -25,7 +25,7 @@ CREATE TABLE master (id Int32, name String) ENGINE = Join (ANY, LEFT, id) SETTIN INSERT INTO master VALUES (1, 'ONE'); INSERT INTO transaction VALUES (1, 52.5, 1); -SELECT tx.id, tx.value, m.name FROM transaction tx ANY LEFT JOIN master m ON m.id = tx.master_id; +SELECT tx.id, tx.value, m.name FROM transaction tx ANY LEFT JOIN master m ON m.id = tx.master_id ORDER BY tx.id; DROP TABLE master; DROP TABLE transaction; diff --git a/tests/queries/0_stateless/01051_system_stack_trace.reference b/tests/queries/0_stateless/01051_system_stack_trace.reference index b82bda76142..5142593dba6 100644 --- a/tests/queries/0_stateless/01051_system_stack_trace.reference +++ b/tests/queries/0_stateless/01051_system_stack_trace.reference @@ -5,7 +5,7 @@ SELECT count() > 0 FROM system.stack_trace WHERE query_id != ''; SELECT countIf(thread_id > 0) > 0 FROM system.stack_trace; 1 -- optimization for trace -SELECT length(trace) > 0 FROM system.stack_trace LIMIT 1; +SELECT count(trace) > 0 FROM system.stack_trace WHERE length(trace) > 0 LIMIT 1; 1 -- optimization for query_id SELECT length(query_id) > 0 FROM system.stack_trace WHERE query_id != '' LIMIT 1; diff --git a/tests/queries/0_stateless/01051_system_stack_trace.sql b/tests/queries/0_stateless/01051_system_stack_trace.sql index d018d01fa22..e322462a46a 100644 --- a/tests/queries/0_stateless/01051_system_stack_trace.sql +++ b/tests/queries/0_stateless/01051_system_stack_trace.sql @@ -5,7 +5,7 @@ SELECT count() > 0 FROM system.stack_trace WHERE query_id != ''; -- opimization for not reading /proc/self/task/{}/comm and avoid sending signal SELECT countIf(thread_id > 0) > 0 FROM system.stack_trace; -- optimization for trace -SELECT length(trace) > 0 FROM system.stack_trace LIMIT 1; +SELECT count(trace) > 0 FROM system.stack_trace WHERE length(trace) > 0 LIMIT 1; -- optimization for query_id SELECT length(query_id) > 0 FROM system.stack_trace WHERE query_id != '' LIMIT 1; -- optimization for thread_name diff --git a/tests/queries/0_stateless/01161_all_system_tables.sh b/tests/queries/0_stateless/01161_all_system_tables.sh index 9988c1f3625..1d886374c07 100755 --- a/tests/queries/0_stateless/01161_all_system_tables.sh +++ b/tests/queries/0_stateless/01161_all_system_tables.sh @@ -16,7 +16,7 @@ function run_selects() { thread_num=$1 readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} -q "SELECT database || '.' || name FROM system.tables - WHERE database in ('system', 'information_schema', 'INFORMATION_SCHEMA') and name!='zookeeper' and name!='merge_tree_metadata_cache' + WHERE database in ('system', 'information_schema', 'INFORMATION_SCHEMA') and name!='zookeeper' and name!='merge_tree_metadata_cache' and name!='models' AND sipHash64(name || toString($RAND)) % $THREADS = $thread_num") for t in "${tables_arr[@]}" diff --git a/tests/queries/0_stateless/01233_check_table_with_metadata_cache.sh b/tests/queries/0_stateless/01233_check_table_with_metadata_cache.sh index 8736bc474b1..67f11e58a68 100755 --- a/tests/queries/0_stateless/01233_check_table_with_metadata_cache.sh +++ b/tests/queries/0_stateless/01233_check_table_with_metadata_cache.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Tags: no-fasttest, long, no-s3-storage +# Tags: no-fasttest, long, no-s3-storage, no-random-settings, no-parallel # Tag no-fasttest: setting use_metadata_cache=true is not supported in fasttest, because clickhouse binary in fasttest is build without RocksDB. +# Tag no-random-settings: random settings significantly slow down test with debug build (alternative: add no-debug tag) # To suppress Warning messages from CHECK TABLE CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=error CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 49f4d8245a2..5c9e1a839b2 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -97,6 +97,7 @@ SYSTEM DROP FILESYSTEM CACHE ['SYSTEM DROP FILESYSTEM CACHE','DROP FILESYSTEM CA SYSTEM DROP SCHEMA CACHE ['SYSTEM DROP SCHEMA CACHE','DROP SCHEMA CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP CACHE ['DROP CACHE'] \N SYSTEM SYSTEM RELOAD CONFIG ['RELOAD CONFIG'] GLOBAL SYSTEM RELOAD +SYSTEM RELOAD USERS ['RELOAD USERS'] GLOBAL SYSTEM RELOAD SYSTEM RELOAD SYMBOLS ['RELOAD SYMBOLS'] GLOBAL SYSTEM RELOAD SYSTEM RELOAD DICTIONARY ['SYSTEM RELOAD DICTIONARIES','RELOAD DICTIONARY','RELOAD DICTIONARIES'] GLOBAL SYSTEM RELOAD SYSTEM RELOAD MODEL ['SYSTEM RELOAD MODELS','RELOAD MODEL','RELOAD MODELS'] GLOBAL SYSTEM RELOAD diff --git a/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql index d5f731568d1..2814c87c933 100644 --- a/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql +++ b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql @@ -2,6 +2,7 @@ DROP TABLE IF EXISTS data_01283; set remote_filesystem_read_method = 'read'; set local_filesystem_read_method = 'pread'; +set load_marks_asynchronously = 0; CREATE TABLE data_01283 engine=MergeTree() ORDER BY key diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference index 997a9504bb5..f723412c636 100644 --- a/tests/queries/0_stateless/01292_create_user.reference +++ b/tests/queries/0_stateless/01292_create_user.reference @@ -58,9 +58,9 @@ CREATE USER u2_01292 SETTINGS PROFILE default CREATE USER u3_01292 SETTINGS max_memory_usage = 5000000 CREATE USER u4_01292 SETTINGS max_memory_usage MIN 5000000 CREATE USER u5_01292 SETTINGS max_memory_usage MAX 5000000 -CREATE USER u6_01292 SETTINGS max_memory_usage READONLY +CREATE USER u6_01292 SETTINGS max_memory_usage CONST CREATE USER u7_01292 SETTINGS max_memory_usage WRITABLE -CREATE USER u8_01292 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 READONLY +CREATE USER u8_01292 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST CREATE USER u9_01292 SETTINGS PROFILE default, max_memory_usage = 5000000 WRITABLE CREATE USER u1_01292 SETTINGS readonly = 1 CREATE USER u2_01292 SETTINGS readonly = 1 @@ -111,7 +111,7 @@ u4_01292 local directory double_sha1_password {} ['::/0'] [] [] [] 1 [] ['r1_012 -- system.settings_profile_elements \N u1_01292 \N 0 readonly 1 \N \N \N \N \N u2_01292 \N 0 \N \N \N \N \N default -\N u3_01292 \N 0 max_memory_usage 5000000 4000000 6000000 0 \N +\N u3_01292 \N 0 max_memory_usage 5000000 4000000 6000000 WRITABLE \N \N u4_01292 \N 0 \N \N \N \N \N default \N u4_01292 \N 1 max_memory_usage 5000000 \N \N \N \N \N u4_01292 \N 2 readonly 1 \N \N \N \N diff --git a/tests/queries/0_stateless/01292_create_user.sql b/tests/queries/0_stateless/01292_create_user.sql index a9582376825..d0f157d36b0 100644 --- a/tests/queries/0_stateless/01292_create_user.sql +++ b/tests/queries/0_stateless/01292_create_user.sql @@ -122,9 +122,9 @@ CREATE USER u2_01292 SETTINGS PROFILE 'default'; CREATE USER u3_01292 SETTINGS max_memory_usage=5000000; CREATE USER u4_01292 SETTINGS max_memory_usage MIN=5000000; CREATE USER u5_01292 SETTINGS max_memory_usage MAX=5000000; -CREATE USER u6_01292 SETTINGS max_memory_usage READONLY; +CREATE USER u6_01292 SETTINGS max_memory_usage CONST; CREATE USER u7_01292 SETTINGS max_memory_usage WRITABLE; -CREATE USER u8_01292 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY; +CREATE USER u8_01292 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 CONST; CREATE USER u9_01292 SETTINGS PROFILE 'default', max_memory_usage=5000000 WRITABLE; SHOW CREATE USER u1_01292; SHOW CREATE USER u2_01292; diff --git a/tests/queries/0_stateless/01293_create_role.reference b/tests/queries/0_stateless/01293_create_role.reference index 8d9a259ecf5..9b3c4eabd47 100644 --- a/tests/queries/0_stateless/01293_create_role.reference +++ b/tests/queries/0_stateless/01293_create_role.reference @@ -15,9 +15,9 @@ CREATE ROLE r2_01293 SETTINGS PROFILE default CREATE ROLE r3_01293 SETTINGS max_memory_usage = 5000000 CREATE ROLE r4_01293 SETTINGS max_memory_usage MIN 5000000 CREATE ROLE r5_01293 SETTINGS max_memory_usage MAX 5000000 -CREATE ROLE r6_01293 SETTINGS max_memory_usage READONLY +CREATE ROLE r6_01293 SETTINGS max_memory_usage CONST CREATE ROLE r7_01293 SETTINGS max_memory_usage WRITABLE -CREATE ROLE r8_01293 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 READONLY +CREATE ROLE r8_01293 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST CREATE ROLE r9_01293 SETTINGS PROFILE default, max_memory_usage = 5000000 WRITABLE CREATE ROLE r1_01293 SETTINGS readonly = 1 CREATE ROLE r2_01293 SETTINGS readonly = 1 @@ -32,7 +32,7 @@ r1_01293 local directory -- system.settings_profile_elements \N \N r1_01293 0 readonly 1 \N \N \N \N \N \N r2_01293 0 \N \N \N \N \N default -\N \N r3_01293 0 max_memory_usage 5000000 4000000 6000000 0 \N +\N \N r3_01293 0 max_memory_usage 5000000 4000000 6000000 WRITABLE \N \N \N r4_01293 0 \N \N \N \N \N default \N \N r4_01293 1 max_memory_usage 5000000 \N \N \N \N \N \N r4_01293 2 readonly 1 \N \N \N \N diff --git a/tests/queries/0_stateless/01293_create_role.sql b/tests/queries/0_stateless/01293_create_role.sql index 963a1020e3f..f22edfeec3e 100644 --- a/tests/queries/0_stateless/01293_create_role.sql +++ b/tests/queries/0_stateless/01293_create_role.sql @@ -31,9 +31,9 @@ CREATE ROLE r2_01293 SETTINGS PROFILE 'default'; CREATE ROLE r3_01293 SETTINGS max_memory_usage=5000000; CREATE ROLE r4_01293 SETTINGS max_memory_usage MIN=5000000; CREATE ROLE r5_01293 SETTINGS max_memory_usage MAX=5000000; -CREATE ROLE r6_01293 SETTINGS max_memory_usage READONLY; +CREATE ROLE r6_01293 SETTINGS max_memory_usage CONST; CREATE ROLE r7_01293 SETTINGS max_memory_usage WRITABLE; -CREATE ROLE r8_01293 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY; +CREATE ROLE r8_01293 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 CONST; CREATE ROLE r9_01293 SETTINGS PROFILE 'default', max_memory_usage=5000000 WRITABLE; SHOW CREATE ROLE r1_01293; SHOW CREATE ROLE r2_01293; diff --git a/tests/queries/0_stateless/01294_create_settings_profile.reference b/tests/queries/0_stateless/01294_create_settings_profile.reference index da47b084070..dbb73bca851 100644 --- a/tests/queries/0_stateless/01294_create_settings_profile.reference +++ b/tests/queries/0_stateless/01294_create_settings_profile.reference @@ -11,9 +11,9 @@ CREATE SETTINGS PROFILE s2_01294 SETTINGS INHERIT default CREATE SETTINGS PROFILE s3_01294 SETTINGS max_memory_usage = 5000000 CREATE SETTINGS PROFILE s4_01294 SETTINGS max_memory_usage MIN 5000000 CREATE SETTINGS PROFILE s5_01294 SETTINGS max_memory_usage MAX 5000000 -CREATE SETTINGS PROFILE s6_01294 SETTINGS max_memory_usage READONLY +CREATE SETTINGS PROFILE s6_01294 SETTINGS max_memory_usage CONST CREATE SETTINGS PROFILE s7_01294 SETTINGS max_memory_usage WRITABLE -CREATE SETTINGS PROFILE s8_01294 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 READONLY +CREATE SETTINGS PROFILE s8_01294 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST CREATE SETTINGS PROFILE s9_01294 SETTINGS INHERIT default, max_memory_usage = 5000000 WRITABLE CREATE SETTINGS PROFILE s10_01294 SETTINGS INHERIT s1_01294, INHERIT s3_01294, INHERIT default, readonly = 0, max_memory_usage MAX 6000000 CREATE SETTINGS PROFILE s1_01294 SETTINGS readonly = 0 @@ -47,11 +47,11 @@ CREATE SETTINGS PROFILE s3_01294 TO r1_01294 CREATE SETTINGS PROFILE s4_01294 TO r1_01294 -- readonly ambiguity CREATE SETTINGS PROFILE s1_01294 SETTINGS readonly = 1 -CREATE SETTINGS PROFILE s2_01294 SETTINGS readonly READONLY +CREATE SETTINGS PROFILE s2_01294 SETTINGS readonly CONST CREATE SETTINGS PROFILE s3_01294 SETTINGS INHERIT readonly CREATE SETTINGS PROFILE s4_01294 SETTINGS INHERIT readonly, INHERIT readonly CREATE SETTINGS PROFILE s5_01294 SETTINGS INHERIT readonly, readonly = 1 -CREATE SETTINGS PROFILE s6_01294 SETTINGS INHERIT readonly, readonly READONLY +CREATE SETTINGS PROFILE s6_01294 SETTINGS INHERIT readonly, readonly CONST -- system.settings_profiles s1_01294 local directory 0 0 [] [] s2_01294 local directory 1 0 ['r1_01294'] [] @@ -61,8 +61,8 @@ s5_01294 local directory 3 0 ['u1_01294'] [] s6_01294 local directory 0 1 [] ['r1_01294','u1_01294'] -- system.settings_profile_elements s2_01294 \N \N 0 readonly 0 \N \N \N \N -s3_01294 \N \N 0 max_memory_usage 5000000 4000000 6000000 1 \N +s3_01294 \N \N 0 max_memory_usage 5000000 4000000 6000000 CONST \N s4_01294 \N \N 0 max_memory_usage 5000000 \N \N \N \N s5_01294 \N \N 0 \N \N \N \N \N default s5_01294 \N \N 1 readonly 0 \N \N \N \N -s5_01294 \N \N 2 max_memory_usage \N \N 6000000 0 \N +s5_01294 \N \N 2 max_memory_usage \N \N 6000000 WRITABLE \N diff --git a/tests/queries/0_stateless/01294_create_settings_profile.sql b/tests/queries/0_stateless/01294_create_settings_profile.sql index b7dd91ad6ed..565b4e70367 100644 --- a/tests/queries/0_stateless/01294_create_settings_profile.sql +++ b/tests/queries/0_stateless/01294_create_settings_profile.sql @@ -25,9 +25,9 @@ CREATE PROFILE s2_01294 SETTINGS INHERIT 'default'; CREATE PROFILE s3_01294 SETTINGS max_memory_usage=5000000; CREATE PROFILE s4_01294 SETTINGS max_memory_usage MIN=5000000; CREATE PROFILE s5_01294 SETTINGS max_memory_usage MAX=5000000; -CREATE PROFILE s6_01294 SETTINGS max_memory_usage READONLY; +CREATE PROFILE s6_01294 SETTINGS max_memory_usage CONST; CREATE PROFILE s7_01294 SETTINGS max_memory_usage WRITABLE; -CREATE PROFILE s8_01294 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY; +CREATE PROFILE s8_01294 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 CONST; CREATE PROFILE s9_01294 SETTINGS INHERIT 'default', max_memory_usage=5000000 WRITABLE; CREATE PROFILE s10_01294 SETTINGS INHERIT s1_01294, s3_01294, INHERIT default, readonly=0, max_memory_usage MAX 6000000; SHOW CREATE PROFILE s1_01294; @@ -106,7 +106,7 @@ DROP PROFILE s1_01294, s2_01294, s3_01294, s4_01294, s5_01294, s6_01294; SELECT '-- system.settings_profiles'; CREATE PROFILE s1_01294; CREATE PROFILE s2_01294 SETTINGS readonly=0 TO r1_01294;; -CREATE PROFILE s3_01294 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY TO r1_01294; +CREATE PROFILE s3_01294 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 CONST TO r1_01294; CREATE PROFILE s4_01294 SETTINGS max_memory_usage=5000000 TO r1_01294; CREATE PROFILE s5_01294 SETTINGS INHERIT default, readonly=0, max_memory_usage MAX 6000000 WRITABLE TO u1_01294; CREATE PROFILE s6_01294 TO ALL EXCEPT u1_01294, r1_01294; diff --git a/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.reference b/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.reference new file mode 100644 index 00000000000..0c720206065 --- /dev/null +++ b/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.reference @@ -0,0 +1,2 @@ +2020-01-01 01:00:00 1 +2020-01-01 01:00:00 999 diff --git a/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.sql b/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.sql new file mode 100644 index 00000000000..4aa52fe6ae8 --- /dev/null +++ b/tests/queries/0_stateless/01321_monotonous_functions_in_order_by_bug.sql @@ -0,0 +1,7 @@ +SELECT + toStartOfHour(c1) AS _c1, + c2 +FROM values((toDateTime('2020-01-01 01:01:01'), 999), (toDateTime('2020-01-01 01:01:59'), 1)) +ORDER BY + _c1 ASC, + c2 ASC diff --git a/tests/queries/0_stateless/01323_too_many_threads_bug.sql b/tests/queries/0_stateless/01323_too_many_threads_bug.sql index d3254d49728..c2cce81d200 100644 --- a/tests/queries/0_stateless/01323_too_many_threads_bug.sql +++ b/tests/queries/0_stateless/01323_too_many_threads_bug.sql @@ -2,6 +2,7 @@ drop table if exists table_01323_many_parts; set remote_filesystem_read_method = 'read'; set local_filesystem_read_method = 'pread'; +set load_marks_asynchronously = 0; create table table_01323_many_parts (x UInt64) engine = MergeTree order by x partition by x % 100; set max_partitions_per_insert_block = 100; diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose.sh b/tests/queries/0_stateless/01417_freeze_partition_verbose.sh index 12f104b5337..1af700c1f6e 100755 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose.sh +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-replicated-database, no-parallel, no-ordinary-database +# Tags: no-replicated-database, no-parallel # Tag no-replicated-database: Unsupported type of ALTER query CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql b/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql index 23678c1abd9..dafe652d271 100644 --- a/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql +++ b/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql @@ -33,6 +33,7 @@ OPTIMIZE TABLE select_final FINAL; SET remote_filesystem_read_method = 'read'; SET local_filesystem_read_method = 'pread'; +set load_marks_asynchronously = 0; SELECT max(x) FROM select_final FINAL; diff --git a/tests/queries/0_stateless/01576_alias_column_rewrite.sql b/tests/queries/0_stateless/01576_alias_column_rewrite.sql index 8424eb11f9b..1f28225bef8 100644 --- a/tests/queries/0_stateless/01576_alias_column_rewrite.sql +++ b/tests/queries/0_stateless/01576_alias_column_rewrite.sql @@ -17,7 +17,7 @@ INSERT INTO test_table(timestamp, value) SELECT toDateTime('2020-01-01 12:00:00' INSERT INTO test_table(timestamp, value) SELECT toDateTime('2020-01-02 12:00:00'), 1 FROM numbers(10); INSERT INTO test_table(timestamp, value) SELECT toDateTime('2020-01-03 12:00:00'), 1 FROM numbers(10); -set optimize_respect_aliases = 1; +set optimize_respect_aliases = 1, optimize_monotonous_functions_in_order_by = 1; SELECT 'test-partition-prune'; SELECT COUNT() = 10 FROM test_table WHERE day = '2020-01-01' SETTINGS max_rows_to_read = 10; diff --git a/tests/queries/0_stateless/01646_rewrite_sum_if_bug.reference b/tests/queries/0_stateless/01646_rewrite_sum_if_bug.reference new file mode 100644 index 00000000000..dda2df4fd48 --- /dev/null +++ b/tests/queries/0_stateless/01646_rewrite_sum_if_bug.reference @@ -0,0 +1,2 @@ +67 +0 100 diff --git a/tests/queries/0_stateless/01646_rewrite_sum_if_bug.sql b/tests/queries/0_stateless/01646_rewrite_sum_if_bug.sql new file mode 100644 index 00000000000..3e6a7b92dbf --- /dev/null +++ b/tests/queries/0_stateless/01646_rewrite_sum_if_bug.sql @@ -0,0 +1,26 @@ +DROP TABLE IF EXISTS t; +create table t( s String ) Engine=Memory as select arrayJoin (['a','b','c']); + +SELECT round((sum(multiIf(s IN ('a', 'b'), 1, 0)) / count()) * 100) AS r +FROM cluster('test_cluster_two_shards', currentDatabase(), t); + +DROP TABLE t; + + +DROP TABLE IF EXISTS test_alias; + +CREATE TABLE test_alias(`a` Int64, `b` Int64, `c` Int64, `day` Date, `rtime` DateTime) ENGINE = Memory +as select 0, 0, 0, '2022-01-01', 0 from zeros(10); + +WITH + sum(if((a >= 0) AND (b != 100) AND (c = 0), 1, 0)) AS r1, + sum(if((a >= 0) AND (b != 100) AND (c > 220), 1, 0)) AS r2 +SELECT + (intDiv(toUInt32(rtime), 20) * 20) * 1000 AS t, + (r1 * 100) / (r1 + r2) AS m +FROM cluster('test_cluster_two_shards', currentDatabase(), test_alias) +WHERE day = '2022-01-01' +GROUP BY t +ORDER BY t ASC; + +DROP TABLE test_alias; diff --git a/tests/queries/0_stateless/01655_window_functions_bug.reference b/tests/queries/0_stateless/01655_window_functions_bug.reference new file mode 100644 index 00000000000..2d44832e1f5 --- /dev/null +++ b/tests/queries/0_stateless/01655_window_functions_bug.reference @@ -0,0 +1,3 @@ +0 1 +-2 2 +-2 2 diff --git a/tests/queries/0_stateless/01655_window_functions_bug.sql b/tests/queries/0_stateless/01655_window_functions_bug.sql new file mode 100644 index 00000000000..8d0b2fc62e5 --- /dev/null +++ b/tests/queries/0_stateless/01655_window_functions_bug.sql @@ -0,0 +1,5 @@ +SELECT round((countIf(rating = 5)) - (countIf(rating < 5)), 4) as nps, + dense_rank() OVER (ORDER BY nps DESC) as rank +FROM (select number as rating, number%3 rest_id from numbers(10)) +group by rest_id +order by rank; diff --git a/tests/queries/0_stateless/01702_system_query_log.reference b/tests/queries/0_stateless/01702_system_query_log.reference index 1f329feac22..4b9eeb139f4 100644 --- a/tests/queries/0_stateless/01702_system_query_log.reference +++ b/tests/queries/0_stateless/01702_system_query_log.reference @@ -21,7 +21,7 @@ Create CREATE DATABASE sqllt; Create CREATE TABLE sqllt.table\n(\n i UInt8, s String\n)\nENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple(); Create CREATE VIEW sqllt.view AS SELECT i, s FROM sqllt.table; Create CREATE DICTIONARY sqllt.dictionary (key UInt64, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(DB \'sqllt\' TABLE \'table\' HOST \'localhost\' PORT 9001)) LIFETIME(0) LAYOUT(FLAT()); - CREATE USER sqllt_user IDENTIFIED WITH PLAINTEXT_PASSWORD BY \'password\'; + CREATE USER sqllt_user IDENTIFIED WITH plaintext_password CREATE ROLE sqllt_role; CREATE POLICY sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary AS PERMISSIVE TO ALL; CREATE POLICY sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary AS PERMISSIVE TO ALL; diff --git a/tests/queries/0_stateless/01799_long_uniq_theta_sketch.reference b/tests/queries/0_stateless/01799_long_uniq_theta_sketch.reference index 2bba987fa79..b0234c47b8c 100644 --- a/tests/queries/0_stateless/01799_long_uniq_theta_sketch.reference +++ b/tests/queries/0_stateless/01799_long_uniq_theta_sketch.reference @@ -52,110 +52,110 @@ uniqTheta 35 52331 36 53766 uniqTheta round(float) -0.125 1 -0.5 1 -0.05 1 -0.143 1 -0.056 1 -0.048 2 -0.083 1 -0.25 1 -0.1 1 -0.028 1 0.027 1 +0.028 1 0.031 1 -0.067 1 0.037 1 -0.045 162 -0.125 163 -0.5 162 -0.05 162 -0.143 162 -0.091 81 -0.056 162 -0.048 162 -0.083 163 -0.25 162 -1 162 -0.1 163 -0.028 162 +0.048 2 +0.05 1 +0.056 1 +0.067 1 +0.083 1 +0.1 1 +0.125 1 +0.143 1 +0.25 1 +0.5 1 0.027 162 +0.028 162 0.031 162 -0.067 162 -0.043 162 0.037 162 +0.043 162 +0.045 162 +0.048 162 +0.05 162 +0.056 162 +0.067 162 0.071 162 +0.083 163 +0.091 81 +0.1 163 +0.125 163 +0.143 162 +0.25 162 +0.5 162 +1 162 +0.027 53766 +0.028 52331 +0.031 54139 +0.037 53716 +0.043 54690 0.045 53054 -0.125 53839 -0.5 54020 -0.05 53680 -0.143 53947 -0.091 26876 -0.056 53331 0.048 54211 +0.05 53680 +0.056 53331 +0.067 53516 +0.071 53479 0.083 54985 +0.091 26876 +0.1 54408 +0.125 53839 +0.143 53947 0.25 53774 +0.5 54020 1 55018 -0.1 54408 -0.028 52331 -0.027 53766 -0.031 54139 -0.067 53516 -0.043 54690 -0.037 53716 -0.071 53479 uniqTheta round(toFloat32()) -0.5 1 -0.05 1 -0.25 1 -0.048 2 -0.083 1 -0.125 1 -0.031 1 -0.143 1 -0.028 1 -0.067 1 0.027 1 -0.056 1 +0.028 1 +0.031 1 0.037 1 +0.048 2 +0.05 1 +0.056 1 +0.067 1 +0.083 1 0.1 1 -0.5 162 -0.05 162 -0.25 162 -0.048 162 -0.091 81 +0.125 1 +0.143 1 +0.25 1 +0.5 1 +0.027 162 +0.028 162 +0.031 162 +0.037 162 0.043 162 +0.045 162 +0.048 162 +0.05 162 +0.056 162 +0.067 162 0.071 162 0.083 163 -0.125 163 -0.031 162 -0.143 162 -0.028 162 -0.067 162 -0.045 162 -0.027 162 -0.056 162 -0.037 162 +0.091 81 0.1 163 +0.125 163 +0.143 162 +0.25 162 +0.5 162 1 162 -0.5 54020 -0.05 53680 -0.25 53774 -0.048 54211 -0.091 26876 +0.027 53766 +0.028 52331 +0.031 54139 +0.037 53716 0.043 54690 +0.045 53054 +0.048 54211 +0.05 53680 +0.056 53331 +0.067 53516 0.071 53479 0.083 54985 -0.125 53839 -0.031 54139 -0.143 53947 -0.028 52331 -0.067 53516 -0.045 53054 -0.027 53766 -0.056 53331 -0.037 53716 +0.091 26876 0.1 54408 +0.125 53839 +0.143 53947 +0.25 53774 +0.5 54020 1 55018 uniqTheta IPv4NumToString 1 1 diff --git a/tests/queries/0_stateless/01799_long_uniq_theta_sketch.sql b/tests/queries/0_stateless/01799_long_uniq_theta_sketch.sql index d86575c3f0a..37f0c31ab10 100644 --- a/tests/queries/0_stateless/01799_long_uniq_theta_sketch.sql +++ b/tests/queries/0_stateless/01799_long_uniq_theta_sketch.sql @@ -2,27 +2,27 @@ SELECT 'uniqTheta'; -SELECT Y, uniqTheta(X) FROM (SELECT number AS X, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 15) GROUP BY Y; -SELECT Y, uniqTheta(X) FROM (SELECT number AS X, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 3000) GROUP BY Y; -SELECT Y, uniqTheta(X) FROM (SELECT number AS X, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 1000000) GROUP BY Y; +SELECT Y, uniqTheta(X) FROM (SELECT number AS X, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 15) GROUP BY Y ORDER BY Y; +SELECT Y, uniqTheta(X) FROM (SELECT number AS X, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 3000) GROUP BY Y ORDER BY Y; +SELECT Y, uniqTheta(X) FROM (SELECT number AS X, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 1000000) GROUP BY Y ORDER BY Y; SELECT 'uniqTheta round(float)'; -SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(1/(1 + (3*X*X - 7*X + 11) % 37), 3) AS Y FROM system.numbers LIMIT 15) GROUP BY Y; -SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(1/(1 + (3*X*X - 7*X + 11) % 37), 3) AS Y FROM system.numbers LIMIT 3000) GROUP BY Y; -SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(1/(1 + (3*X*X - 7*X + 11) % 37), 3) AS Y FROM system.numbers LIMIT 1000000) GROUP BY Y; +SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(1/(1 + (3*X*X - 7*X + 11) % 37), 3) AS Y FROM system.numbers LIMIT 15) GROUP BY Y ORDER BY Y; +SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(1/(1 + (3*X*X - 7*X + 11) % 37), 3) AS Y FROM system.numbers LIMIT 3000) GROUP BY Y ORDER BY Y; +SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(1/(1 + (3*X*X - 7*X + 11) % 37), 3) AS Y FROM system.numbers LIMIT 1000000) GROUP BY Y ORDER BY Y; SELECT 'uniqTheta round(toFloat32())'; -SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(toFloat32(1/(1 + (3*X*X - 7*X + 11) % 37)), 3) AS Y FROM system.numbers LIMIT 15) GROUP BY Y; -SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(toFloat32(1/(1 + (3*X*X - 7*X + 11) % 37)), 3) AS Y FROM system.numbers LIMIT 3000) GROUP BY Y; -SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(toFloat32(1/(1 + (3*X*X - 7*X + 11) % 37)), 3) AS Y FROM system.numbers LIMIT 1000000) GROUP BY Y; +SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(toFloat32(1/(1 + (3*X*X - 7*X + 11) % 37)), 3) AS Y FROM system.numbers LIMIT 15) GROUP BY Y ORDER BY Y; +SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(toFloat32(1/(1 + (3*X*X - 7*X + 11) % 37)), 3) AS Y FROM system.numbers LIMIT 3000) GROUP BY Y ORDER BY Y; +SELECT Y, uniqTheta(X) FROM (SELECT number AS X, round(toFloat32(1/(1 + (3*X*X - 7*X + 11) % 37)), 3) AS Y FROM system.numbers LIMIT 1000000) GROUP BY Y ORDER BY Y; SELECT 'uniqTheta IPv4NumToString'; -SELECT Y, uniqTheta(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 15) GROUP BY Y; -SELECT Y, uniqTheta(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 3000) GROUP BY Y; -SELECT Y, uniqTheta(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 1000000) GROUP BY Y; +SELECT Y, uniqTheta(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 15) GROUP BY Y ORDER BY Y; +SELECT Y, uniqTheta(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 3000) GROUP BY Y ORDER BY Y; +SELECT Y, uniqTheta(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 1000000) GROUP BY Y ORDER BY Y; SELECT 'uniqTheta remote()'; diff --git a/tests/queries/0_stateless/01825_type_json_add_column.reference.j2 b/tests/queries/0_stateless/01825_type_json_add_column.reference.j2 new file mode 100644 index 00000000000..da724aef01a --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_add_column.reference.j2 @@ -0,0 +1,6 @@ +{% for storage in ["MergeTree", "ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_add_column/', 'r1')"] -%} +{"id":"1","s":{"k1":0}} +{"id":"2","s":{"k1":100}} +{"id":"1"} +{"id":"2"} +{% endfor -%} diff --git a/tests/queries/0_stateless/01825_type_json_add_column.sql.j2 b/tests/queries/0_stateless/01825_type_json_add_column.sql.j2 new file mode 100644 index 00000000000..87c76c042a6 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_add_column.sql.j2 @@ -0,0 +1,23 @@ +-- Tags: no-fasttest + +{% for storage in ["MergeTree", "ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_add_column/', 'r1')"] -%} + +DROP TABLE IF EXISTS t_json_add_column; +SET allow_experimental_object_type = 1; + +CREATE TABLE t_json_add_column (id UInt64) ENGINE = {{ storage }} ORDER BY tuple(); + +INSERT INTO t_json_add_column VALUES (1); +ALTER TABLE t_json_add_column ADD COLUMN s JSON; + +INSERT INTO t_json_add_column VALUES(2, '{"k1": 100}'); + +SELECT * FROM t_json_add_column ORDER BY id FORMAT JSONEachRow; + +ALTER TABLE t_json_add_column DROP COLUMN s; + +SELECT * FROM t_json_add_column ORDER BY id FORMAT JSONEachRow; + +DROP TABLE t_json_add_column; + +{% endfor -%} diff --git a/tests/queries/0_stateless/01825_type_json_sparse.sql b/tests/queries/0_stateless/01825_type_json_sparse.sql index 2e7520f3e7a..343013cb3da 100644 --- a/tests/queries/0_stateless/01825_type_json_sparse.sql +++ b/tests/queries/0_stateless/01825_type_json_sparse.sql @@ -18,7 +18,8 @@ SELECT subcolumns.names, subcolumns.serializations, count() FROM system.parts_co ARRAY JOIN subcolumns WHERE database = currentDatabase() AND table = 't_json_sparse' AND column = 'data' AND active -GROUP BY subcolumns.names, subcolumns.serializations; +GROUP BY subcolumns.names, subcolumns.serializations +ORDER BY subcolumns.names; SELECT '============='; @@ -29,7 +30,8 @@ SELECT subcolumns.names, subcolumns.serializations, count() FROM system.parts_co ARRAY JOIN subcolumns WHERE database = currentDatabase() AND table = 't_json_sparse' AND column = 'data' AND active -GROUP BY subcolumns.names, subcolumns.serializations; +GROUP BY subcolumns.names, subcolumns.serializations +ORDER BY subcolumns.names; SELECT '============='; @@ -40,7 +42,8 @@ SELECT subcolumns.names, subcolumns.serializations, count() FROM system.parts_co ARRAY JOIN subcolumns WHERE database = currentDatabase() AND table = 't_json_sparse' AND column = 'data' AND active -GROUP BY subcolumns.names, subcolumns.serializations; +GROUP BY subcolumns.names, subcolumns.serializations +ORDER BY subcolumns.names; INSERT INTO t_json_sparse SELECT '{"k1": 2}' FROM numbers(200000); @@ -52,8 +55,9 @@ SELECT subcolumns.names, subcolumns.serializations, count() FROM system.parts_co ARRAY JOIN subcolumns WHERE database = currentDatabase() AND table = 't_json_sparse' AND column = 'data' AND active -GROUP BY subcolumns.names, subcolumns.serializations; +GROUP BY subcolumns.names, subcolumns.serializations +ORDER BY subcolumns.names; -SELECT data.k1, count(), sum(data.k2.k3) FROM t_json_sparse GROUP BY data.k1; +SELECT data.k1, count(), sum(data.k2.k3) FROM t_json_sparse GROUP BY data.k1 ORDER BY data.k1; -DROP TABLE t_json_sparse; +-- DROP TABLE t_json_sparse; diff --git a/tests/queries/0_stateless/01921_datatype_date32.reference b/tests/queries/0_stateless/01921_datatype_date32.reference index a33a96ffffb..b5bf4e06a4c 100644 --- a/tests/queries/0_stateless/01921_datatype_date32.reference +++ b/tests/queries/0_stateless/01921_datatype_date32.reference @@ -109,10 +109,10 @@ -------toStartOfFifteenMinutes--------- -------toStartOfHour--------- -------toStartOfISOYear--------- -2079-06-07 -2079-06-07 -2119-07-29 -2119-07-29 +1970-01-01 +1970-01-01 +2148-12-30 +2148-12-30 2021-01-04 -------toRelativeYearNum--------- 1900 diff --git a/tests/queries/0_stateless/02006_test_positional_arguments.reference b/tests/queries/0_stateless/02006_test_positional_arguments.reference index 293cc1c7591..56817961b30 100644 --- a/tests/queries/0_stateless/02006_test_positional_arguments.reference +++ b/tests/queries/0_stateless/02006_test_positional_arguments.reference @@ -16,13 +16,13 @@ select x3, x2, x1 from test order by x3 desc; 10 1 10 1 100 100 insert into test values (1, 10, 100), (10, 1, 10), (100, 100, 1); -select x3, x2 from test group by x3, x2; -10 1 +select x3, x2 from test group by x3, x2 order by x3; 1 100 +10 1 100 10 -select x3, x2 from test group by 1, 2; -10 1 +select x3, x2 from test group by 1, 2 order by x3; 1 100 +10 1 100 10 select x1, x2, x3 from test order by x3 limit 1 by x3; 100 100 1 @@ -102,14 +102,14 @@ select x1, x1 * 2, max(x2), max(x3) from test2 group by 2, 1, x1 order by 1, 2, 1 2 10 100 10 20 1 10 100 200 100 1 -select a, b, c, d, e, f from (select 44 a, 88 b, 13 c, 14 d, 15 e, 16 f) t group by 1,2,3,4,5,6; +select a, b, c, d, e, f from (select 44 a, 88 b, 13 c, 14 d, 15 e, 16 f) t group by 1,2,3,4,5,6 order by a; 44 88 13 14 15 16 explain syntax select plus(1, 1) as a group by a; SELECT 1 + 1 AS a GROUP BY a -select substr('aaaaaaaaaaaaaa', 8) as a group by a; +select substr('aaaaaaaaaaaaaa', 8) as a group by a order by a; aaaaaaa -select substr('aaaaaaaaaaaaaa', 8) as a group by substr('aaaaaaaaaaaaaa', 8); +select substr('aaaaaaaaaaaaaa', 8) as a group by substr('aaaaaaaaaaaaaa', 8) order by a; aaaaaaa select b from (select 5 as a, 'Hello' as b order by a); Hello diff --git a/tests/queries/0_stateless/02006_test_positional_arguments.sql b/tests/queries/0_stateless/02006_test_positional_arguments.sql index 71fd1a5c727..8829a204ab6 100644 --- a/tests/queries/0_stateless/02006_test_positional_arguments.sql +++ b/tests/queries/0_stateless/02006_test_positional_arguments.sql @@ -15,8 +15,8 @@ select x3, x2, x1 from test order by 1 desc; select x3, x2, x1 from test order by x3 desc; insert into test values (1, 10, 100), (10, 1, 10), (100, 100, 1); -select x3, x2 from test group by x3, x2; -select x3, x2 from test group by 1, 2; +select x3, x2 from test group by x3, x2 order by x3; +select x3, x2 from test group by 1, 2 order by x3; select x1, x2, x3 from test order by x3 limit 1 by x3; select x1, x2, x3 from test order by 3 limit 1 by 3; @@ -39,11 +39,11 @@ create table test2(x1 Int, x2 Int, x3 Int) engine=Memory; insert into test2 values (1, 10, 100), (10, 1, 10), (100, 100, 1); select x1, x1 * 2, max(x2), max(x3) from test2 group by 2, 1, x1 order by 1, 2, 4 desc, 3 asc; -select a, b, c, d, e, f from (select 44 a, 88 b, 13 c, 14 d, 15 e, 16 f) t group by 1,2,3,4,5,6; +select a, b, c, d, e, f from (select 44 a, 88 b, 13 c, 14 d, 15 e, 16 f) t group by 1,2,3,4,5,6 order by a; explain syntax select plus(1, 1) as a group by a; -select substr('aaaaaaaaaaaaaa', 8) as a group by a; -select substr('aaaaaaaaaaaaaa', 8) as a group by substr('aaaaaaaaaaaaaa', 8); +select substr('aaaaaaaaaaaaaa', 8) as a group by a order by a; +select substr('aaaaaaaaaaaaaa', 8) as a group by substr('aaaaaaaaaaaaaa', 8) order by a; select b from (select 5 as a, 'Hello' as b order by a); select b from (select 5 as a, 'Hello' as b group by a); diff --git a/tests/integration/test_catboost_model_reload/__init__.py b/tests/queries/0_stateless/02070_join_on_disk.reference similarity index 100% rename from tests/integration/test_catboost_model_reload/__init__.py rename to tests/queries/0_stateless/02070_join_on_disk.reference diff --git a/tests/queries/0_stateless/02070_join_on_disk.sql b/tests/queries/0_stateless/02070_join_on_disk.sql new file mode 100644 index 00000000000..eabf31df25f --- /dev/null +++ b/tests/queries/0_stateless/02070_join_on_disk.sql @@ -0,0 +1,21 @@ +-- Regression test when Join stores data on disk and receive empty block. +-- Because of this it does not create empty file, while expect it. + +SET max_threads = 1; +SET join_algorithm = 'auto'; +SET max_rows_in_join = 1000; +SET optimize_aggregation_in_order = 1; +SET max_block_size = 1000; + +DROP TABLE IF EXISTS join_on_disk; + +SYSTEM STOP MERGES join_on_disk; + +CREATE TABLE join_on_disk (id Int) Engine=MergeTree() ORDER BY id; + +INSERT INTO join_on_disk SELECT number as id FROM numbers_mt(50000); +INSERT INTO join_on_disk SELECT number as id FROM numbers_mt(1000); + +SELECT id FROM join_on_disk lhs LEFT JOIN (SELECT id FROM join_on_disk GROUP BY id) rhs USING (id) FORMAT Null; + +DROP TABLE join_on_disk; diff --git a/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.reference b/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.reference new file mode 100644 index 00000000000..2b3f8138c2b --- /dev/null +++ b/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.reference @@ -0,0 +1,11 @@ +-- { echoOn } +-- NOTE: total string size should be > 16 (sizeof(__m128i)) +insert into utf8_overlap values ('\xe2'), ('Foo⚊BarBazBam'), ('\xe2'), ('Foo⚊BarBazBam'); +-- ^ +-- MONOGRAM FOR YANG +with lowerUTF8(str) as l_, upperUTF8(str) as u_, '0x' || hex(str) as h_ +select length(str), if(l_ == '\xe2', h_, l_), if(u_ == '\xe2', h_, u_) from utf8_overlap format CSV; +1,"0xE2","0xE2" +15,"foo⚊barbazbam","FOO⚊BARBAZBAM" +1,"0xE2","0xE2" +15,"foo⚊barbazbam","FOO⚊BARBAZBAM" diff --git a/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.sql b/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.sql new file mode 100644 index 00000000000..ee0d29be699 --- /dev/null +++ b/tests/queries/0_stateless/02071_lower_upper_utf8_row_overlaps.sql @@ -0,0 +1,10 @@ +drop table if exists utf8_overlap; +create table utf8_overlap (str String) engine=Memory(); + +-- { echoOn } +-- NOTE: total string size should be > 16 (sizeof(__m128i)) +insert into utf8_overlap values ('\xe2'), ('Foo⚊BarBazBam'), ('\xe2'), ('Foo⚊BarBazBam'); +-- ^ +-- MONOGRAM FOR YANG +with lowerUTF8(str) as l_, upperUTF8(str) as u_, '0x' || hex(str) as h_ +select length(str), if(l_ == '\xe2', h_, l_), if(u_ == '\xe2', h_, u_) from utf8_overlap format CSV; diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 9e2f676bb55..890505fa585 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -279,7 +279,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD SYMBOLS' = 99, 'SYSTEM RELOAD DICTIONARY' = 100, 'SYSTEM RELOAD MODEL' = 101, 'SYSTEM RELOAD FUNCTION' = 102, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 103, 'SYSTEM RELOAD' = 104, 'SYSTEM RESTART DISK' = 105, 'SYSTEM MERGES' = 106, 'SYSTEM TTL MERGES' = 107, 'SYSTEM FETCHES' = 108, 'SYSTEM MOVES' = 109, 'SYSTEM DISTRIBUTED SENDS' = 110, 'SYSTEM REPLICATED SENDS' = 111, 'SYSTEM SENDS' = 112, 'SYSTEM REPLICATION QUEUES' = 113, 'SYSTEM DROP REPLICA' = 114, 'SYSTEM SYNC REPLICA' = 115, 'SYSTEM RESTART REPLICA' = 116, 'SYSTEM RESTORE REPLICA' = 117, 'SYSTEM SYNC DATABASE REPLICA' = 118, 'SYSTEM SYNC TRANSACTION LOG' = 119, 'SYSTEM FLUSH DISTRIBUTED' = 120, 'SYSTEM FLUSH LOGS' = 121, 'SYSTEM FLUSH' = 122, 'SYSTEM THREAD FUZZER' = 123, 'SYSTEM UNFREEZE' = 124, 'SYSTEM' = 125, 'dictGet' = 126, 'addressToLine' = 127, 'addressToLineWithInlines' = 128, 'addressToSymbol' = 129, 'demangle' = 130, 'INTROSPECTION' = 131, 'FILE' = 132, 'URL' = 133, 'REMOTE' = 134, 'MONGO' = 135, 'MEILISEARCH' = 136, 'MYSQL' = 137, 'POSTGRES' = 138, 'SQLITE' = 139, 'ODBC' = 140, 'JDBC' = 141, 'HDFS' = 142, 'S3' = 143, 'HIVE' = 144, 'SOURCES' = 145, 'CLUSTER' = 146, 'ALL' = 147, 'NONE' = 148), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD USERS' = 99, 'SYSTEM RELOAD SYMBOLS' = 100, 'SYSTEM RELOAD DICTIONARY' = 101, 'SYSTEM RELOAD MODEL' = 102, 'SYSTEM RELOAD FUNCTION' = 103, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 104, 'SYSTEM RELOAD' = 105, 'SYSTEM RESTART DISK' = 106, 'SYSTEM MERGES' = 107, 'SYSTEM TTL MERGES' = 108, 'SYSTEM FETCHES' = 109, 'SYSTEM MOVES' = 110, 'SYSTEM DISTRIBUTED SENDS' = 111, 'SYSTEM REPLICATED SENDS' = 112, 'SYSTEM SENDS' = 113, 'SYSTEM REPLICATION QUEUES' = 114, 'SYSTEM DROP REPLICA' = 115, 'SYSTEM SYNC REPLICA' = 116, 'SYSTEM RESTART REPLICA' = 117, 'SYSTEM RESTORE REPLICA' = 118, 'SYSTEM SYNC DATABASE REPLICA' = 119, 'SYSTEM SYNC TRANSACTION LOG' = 120, 'SYSTEM FLUSH DISTRIBUTED' = 121, 'SYSTEM FLUSH LOGS' = 122, 'SYSTEM FLUSH' = 123, 'SYSTEM THREAD FUZZER' = 124, 'SYSTEM UNFREEZE' = 125, 'SYSTEM' = 126, 'dictGet' = 127, 'addressToLine' = 128, 'addressToLineWithInlines' = 129, 'addressToSymbol' = 130, 'demangle' = 131, 'INTROSPECTION' = 132, 'FILE' = 133, 'URL' = 134, 'REMOTE' = 135, 'MONGO' = 136, 'MEILISEARCH' = 137, 'MYSQL' = 138, 'POSTGRES' = 139, 'SQLITE' = 140, 'ODBC' = 141, 'JDBC' = 142, 'HDFS' = 143, 'S3' = 144, 'HIVE' = 145, 'SOURCES' = 146, 'CLUSTER' = 147, 'ALL' = 148, 'NONE' = 149), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -364,18 +364,6 @@ CREATE TABLE system.metrics ) ENGINE = SystemMetrics COMMENT 'SYSTEM TABLE is built on the fly.' -CREATE TABLE system.models -( - `name` String, - `status` Enum8('NOT_LOADED' = 0, 'LOADED' = 1, 'FAILED' = 2, 'LOADING' = 3, 'FAILED_AND_RELOADING' = 4, 'LOADED_AND_RELOADING' = 5, 'NOT_EXIST' = 6), - `origin` String, - `type` String, - `loading_start_time` DateTime, - `loading_duration` Float32, - `last_exception` String -) -ENGINE = SystemModels -COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.mutations ( `database` String, @@ -492,6 +480,7 @@ CREATE TABLE system.parts `removal_tid` Tuple(UInt64, UInt64, UUID), `creation_csn` UInt64, `removal_csn` UInt64, + `has_lightweight_delete` UInt8, `bytes` UInt64, `marks_size` UInt64 ) @@ -553,10 +542,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD SYMBOLS' = 99, 'SYSTEM RELOAD DICTIONARY' = 100, 'SYSTEM RELOAD MODEL' = 101, 'SYSTEM RELOAD FUNCTION' = 102, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 103, 'SYSTEM RELOAD' = 104, 'SYSTEM RESTART DISK' = 105, 'SYSTEM MERGES' = 106, 'SYSTEM TTL MERGES' = 107, 'SYSTEM FETCHES' = 108, 'SYSTEM MOVES' = 109, 'SYSTEM DISTRIBUTED SENDS' = 110, 'SYSTEM REPLICATED SENDS' = 111, 'SYSTEM SENDS' = 112, 'SYSTEM REPLICATION QUEUES' = 113, 'SYSTEM DROP REPLICA' = 114, 'SYSTEM SYNC REPLICA' = 115, 'SYSTEM RESTART REPLICA' = 116, 'SYSTEM RESTORE REPLICA' = 117, 'SYSTEM SYNC DATABASE REPLICA' = 118, 'SYSTEM SYNC TRANSACTION LOG' = 119, 'SYSTEM FLUSH DISTRIBUTED' = 120, 'SYSTEM FLUSH LOGS' = 121, 'SYSTEM FLUSH' = 122, 'SYSTEM THREAD FUZZER' = 123, 'SYSTEM UNFREEZE' = 124, 'SYSTEM' = 125, 'dictGet' = 126, 'addressToLine' = 127, 'addressToLineWithInlines' = 128, 'addressToSymbol' = 129, 'demangle' = 130, 'INTROSPECTION' = 131, 'FILE' = 132, 'URL' = 133, 'REMOTE' = 134, 'MONGO' = 135, 'MEILISEARCH' = 136, 'MYSQL' = 137, 'POSTGRES' = 138, 'SQLITE' = 139, 'ODBC' = 140, 'JDBC' = 141, 'HDFS' = 142, 'S3' = 143, 'HIVE' = 144, 'SOURCES' = 145, 'CLUSTER' = 146, 'ALL' = 147, 'NONE' = 148), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD USERS' = 99, 'SYSTEM RELOAD SYMBOLS' = 100, 'SYSTEM RELOAD DICTIONARY' = 101, 'SYSTEM RELOAD MODEL' = 102, 'SYSTEM RELOAD FUNCTION' = 103, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 104, 'SYSTEM RELOAD' = 105, 'SYSTEM RESTART DISK' = 106, 'SYSTEM MERGES' = 107, 'SYSTEM TTL MERGES' = 108, 'SYSTEM FETCHES' = 109, 'SYSTEM MOVES' = 110, 'SYSTEM DISTRIBUTED SENDS' = 111, 'SYSTEM REPLICATED SENDS' = 112, 'SYSTEM SENDS' = 113, 'SYSTEM REPLICATION QUEUES' = 114, 'SYSTEM DROP REPLICA' = 115, 'SYSTEM SYNC REPLICA' = 116, 'SYSTEM RESTART REPLICA' = 117, 'SYSTEM RESTORE REPLICA' = 118, 'SYSTEM SYNC DATABASE REPLICA' = 119, 'SYSTEM SYNC TRANSACTION LOG' = 120, 'SYSTEM FLUSH DISTRIBUTED' = 121, 'SYSTEM FLUSH LOGS' = 122, 'SYSTEM FLUSH' = 123, 'SYSTEM THREAD FUZZER' = 124, 'SYSTEM UNFREEZE' = 125, 'SYSTEM' = 126, 'dictGet' = 127, 'addressToLine' = 128, 'addressToLineWithInlines' = 129, 'addressToSymbol' = 130, 'demangle' = 131, 'INTROSPECTION' = 132, 'FILE' = 133, 'URL' = 134, 'REMOTE' = 135, 'MONGO' = 136, 'MEILISEARCH' = 137, 'MYSQL' = 138, 'POSTGRES' = 139, 'SQLITE' = 140, 'ODBC' = 141, 'JDBC' = 142, 'HDFS' = 143, 'S3' = 144, 'HIVE' = 145, 'SOURCES' = 146, 'CLUSTER' = 147, 'ALL' = 148, 'NONE' = 149), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD SYMBOLS' = 99, 'SYSTEM RELOAD DICTIONARY' = 100, 'SYSTEM RELOAD MODEL' = 101, 'SYSTEM RELOAD FUNCTION' = 102, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 103, 'SYSTEM RELOAD' = 104, 'SYSTEM RESTART DISK' = 105, 'SYSTEM MERGES' = 106, 'SYSTEM TTL MERGES' = 107, 'SYSTEM FETCHES' = 108, 'SYSTEM MOVES' = 109, 'SYSTEM DISTRIBUTED SENDS' = 110, 'SYSTEM REPLICATED SENDS' = 111, 'SYSTEM SENDS' = 112, 'SYSTEM REPLICATION QUEUES' = 113, 'SYSTEM DROP REPLICA' = 114, 'SYSTEM SYNC REPLICA' = 115, 'SYSTEM RESTART REPLICA' = 116, 'SYSTEM RESTORE REPLICA' = 117, 'SYSTEM SYNC DATABASE REPLICA' = 118, 'SYSTEM SYNC TRANSACTION LOG' = 119, 'SYSTEM FLUSH DISTRIBUTED' = 120, 'SYSTEM FLUSH LOGS' = 121, 'SYSTEM FLUSH' = 122, 'SYSTEM THREAD FUZZER' = 123, 'SYSTEM UNFREEZE' = 124, 'SYSTEM' = 125, 'dictGet' = 126, 'addressToLine' = 127, 'addressToLineWithInlines' = 128, 'addressToSymbol' = 129, 'demangle' = 130, 'INTROSPECTION' = 131, 'FILE' = 132, 'URL' = 133, 'REMOTE' = 134, 'MONGO' = 135, 'MEILISEARCH' = 136, 'MYSQL' = 137, 'POSTGRES' = 138, 'SQLITE' = 139, 'ODBC' = 140, 'JDBC' = 141, 'HDFS' = 142, 'S3' = 143, 'HIVE' = 144, 'SOURCES' = 145, 'CLUSTER' = 146, 'ALL' = 147, 'NONE' = 148)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER TABLE' = 41, 'ALTER DATABASE' = 42, 'ALTER VIEW REFRESH' = 43, 'ALTER VIEW MODIFY QUERY' = 44, 'ALTER VIEW' = 45, 'ALTER' = 46, 'CREATE DATABASE' = 47, 'CREATE TABLE' = 48, 'CREATE VIEW' = 49, 'CREATE DICTIONARY' = 50, 'CREATE TEMPORARY TABLE' = 51, 'CREATE FUNCTION' = 52, 'CREATE' = 53, 'DROP DATABASE' = 54, 'DROP TABLE' = 55, 'DROP VIEW' = 56, 'DROP DICTIONARY' = 57, 'DROP FUNCTION' = 58, 'DROP' = 59, 'TRUNCATE' = 60, 'OPTIMIZE' = 61, 'BACKUP' = 62, 'KILL QUERY' = 63, 'KILL TRANSACTION' = 64, 'MOVE PARTITION BETWEEN SHARDS' = 65, 'CREATE USER' = 66, 'ALTER USER' = 67, 'DROP USER' = 68, 'CREATE ROLE' = 69, 'ALTER ROLE' = 70, 'DROP ROLE' = 71, 'ROLE ADMIN' = 72, 'CREATE ROW POLICY' = 73, 'ALTER ROW POLICY' = 74, 'DROP ROW POLICY' = 75, 'CREATE QUOTA' = 76, 'ALTER QUOTA' = 77, 'DROP QUOTA' = 78, 'CREATE SETTINGS PROFILE' = 79, 'ALTER SETTINGS PROFILE' = 80, 'DROP SETTINGS PROFILE' = 81, 'SHOW USERS' = 82, 'SHOW ROLES' = 83, 'SHOW ROW POLICIES' = 84, 'SHOW QUOTAS' = 85, 'SHOW SETTINGS PROFILES' = 86, 'SHOW ACCESS' = 87, 'ACCESS MANAGEMENT' = 88, 'SYSTEM SHUTDOWN' = 89, 'SYSTEM DROP DNS CACHE' = 90, 'SYSTEM DROP MARK CACHE' = 91, 'SYSTEM DROP UNCOMPRESSED CACHE' = 92, 'SYSTEM DROP MMAP CACHE' = 93, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 94, 'SYSTEM DROP FILESYSTEM CACHE' = 95, 'SYSTEM DROP SCHEMA CACHE' = 96, 'SYSTEM DROP CACHE' = 97, 'SYSTEM RELOAD CONFIG' = 98, 'SYSTEM RELOAD USERS' = 99, 'SYSTEM RELOAD SYMBOLS' = 100, 'SYSTEM RELOAD DICTIONARY' = 101, 'SYSTEM RELOAD MODEL' = 102, 'SYSTEM RELOAD FUNCTION' = 103, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 104, 'SYSTEM RELOAD' = 105, 'SYSTEM RESTART DISK' = 106, 'SYSTEM MERGES' = 107, 'SYSTEM TTL MERGES' = 108, 'SYSTEM FETCHES' = 109, 'SYSTEM MOVES' = 110, 'SYSTEM DISTRIBUTED SENDS' = 111, 'SYSTEM REPLICATED SENDS' = 112, 'SYSTEM SENDS' = 113, 'SYSTEM REPLICATION QUEUES' = 114, 'SYSTEM DROP REPLICA' = 115, 'SYSTEM SYNC REPLICA' = 116, 'SYSTEM RESTART REPLICA' = 117, 'SYSTEM RESTORE REPLICA' = 118, 'SYSTEM SYNC DATABASE REPLICA' = 119, 'SYSTEM SYNC TRANSACTION LOG' = 120, 'SYSTEM FLUSH DISTRIBUTED' = 121, 'SYSTEM FLUSH LOGS' = 122, 'SYSTEM FLUSH' = 123, 'SYSTEM THREAD FUZZER' = 124, 'SYSTEM UNFREEZE' = 125, 'SYSTEM' = 126, 'dictGet' = 127, 'addressToLine' = 128, 'addressToLineWithInlines' = 129, 'addressToSymbol' = 130, 'demangle' = 131, 'INTROSPECTION' = 132, 'FILE' = 133, 'URL' = 134, 'REMOTE' = 135, 'MONGO' = 136, 'MEILISEARCH' = 137, 'MYSQL' = 138, 'POSTGRES' = 139, 'SQLITE' = 140, 'ODBC' = 141, 'JDBC' = 142, 'HDFS' = 143, 'S3' = 144, 'HIVE' = 145, 'SOURCES' = 146, 'CLUSTER' = 147, 'ALL' = 148, 'NONE' = 149)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' @@ -968,7 +957,7 @@ CREATE TABLE system.settings_profile_elements `value` Nullable(String), `min` Nullable(String), `max` Nullable(String), - `readonly` Nullable(UInt8), + `writability` Nullable(Enum8('WRITABLE' = 0, 'CONST' = 1, 'CHANGEABLE_IN_READONLY' = 2)), `inherit_profile` Nullable(String) ) ENGINE = SystemSettingsProfileElements diff --git a/tests/queries/0_stateless/02117_show_create_table_system.sql b/tests/queries/0_stateless/02117_show_create_table_system.sql index 9a5726a0780..8b75ed60eec 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.sql +++ b/tests/queries/0_stateless/02117_show_create_table_system.sql @@ -45,7 +45,6 @@ show create table macros format TSVRaw; show create table merge_tree_settings format TSVRaw; show create table merges format TSVRaw; show create table metrics format TSVRaw; -show create table models format TSVRaw; show create table mutations format TSVRaw; show create table numbers format TSVRaw; show create table numbers_mt format TSVRaw; diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql index 4dfcbb9bf80..44c1c12be35 100644 --- a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql @@ -56,7 +56,13 @@ ENGINE = MergeTree ORDER BY (toStartOfDay(dt), d); INSERT INTO t_read_in_order SELECT toDateTime('2020-10-10 00:00:00') + number, 1 / (number % 100 + 1), number FROM numbers(1000); EXPLAIN PIPELINE SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; -SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; +SELECT * from ( + SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 50000000000 + -- subquery with limit 50000000 to stabilize a test result and prevent order by d pushdown +) order by d limit 5; EXPLAIN PIPELINE SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; -SELECT toStartOfDay(dt) as date, d FROM t_read_in_order WHERE date = '2020-10-10' ORDER BY round(d) LIMIT 5; +SELECT * from ( + SELECT toStartOfDay(dt) as date, d FROM t_read_in_order WHERE date = '2020-10-10' ORDER BY round(d) LIMIT 50000000000 + -- subquery with limit 50000000 to stabilize a test result and prevent order by d pushdown +) order by d limit 5; diff --git a/tests/queries/0_stateless/02185_orc_corrupted_file.sh b/tests/queries/0_stateless/02185_orc_corrupted_file.sh index 7d7a714cccc..c5f5e8710ca 100755 --- a/tests/queries/0_stateless/02185_orc_corrupted_file.sh +++ b/tests/queries/0_stateless/02185_orc_corrupted_file.sh @@ -8,5 +8,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') cp $CUR_DIR/data_orc/corrupted.orc $USER_FILES_PATH/ -${CLICKHOUSE_CLIENT} --query="select * from file('corrupted.orc')" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL' - +${CLICKHOUSE_CLIENT} --query="select * from file('corrupted.orc')" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL' diff --git a/tests/queries/0_stateless/02245_parquet_skip_unknown_type.sh b/tests/queries/0_stateless/02245_parquet_skip_unknown_type.sh index 005c089e434..1e416f23b69 100755 --- a/tests/queries/0_stateless/02245_parquet_skip_unknown_type.sh +++ b/tests/queries/0_stateless/02245_parquet_skip_unknown_type.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, no-parallel CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -12,7 +12,6 @@ DATA_FILE=$USER_FILES_PATH/$FILE_NAME cp $CUR_DIR/data_parquet_bad_column/metadata_0.parquet $DATA_FILE -$CLICKHOUSE_CLIENT -q "desc file(test_02245.parquet)" 2>&1 | grep -qF "CANNOT_EXTRACT_TABLE_STRUCTURE" && echo "OK" || echo "FAIL" +$CLICKHOUSE_CLIENT -q "desc file(test_02245.parquet)" 2>&1 | grep -qF "Cannot extract table structure" && echo "OK" || echo "FAIL" $CLICKHOUSE_CLIENT -q "desc file(test_02245.parquet) settings input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference=1" $CLICKHOUSE_CLIENT -q "select count(*) from file(test_02245.parquet) settings input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference=1" - diff --git a/tests/queries/0_stateless/02263_lazy_mark_load.sh b/tests/queries/0_stateless/02263_lazy_mark_load.sh index b2558b5741c..bf37556bfa6 100755 --- a/tests/queries/0_stateless/02263_lazy_mark_load.sh +++ b/tests/queries/0_stateless/02263_lazy_mark_load.sh @@ -30,7 +30,7 @@ EOF ${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES lazy_mark_test" ${CLICKHOUSE_CLIENT} -q "INSERT INTO lazy_mark_test select number, number % 3, number % 5, number % 10, number % 13, number % 15, number % 17, number % 18, number % 22, number % 25 from numbers(1000000)" ${CLICKHOUSE_CLIENT} -q "SYSTEM DROP MARK CACHE" -${CLICKHOUSE_CLIENT} --log_queries=1 --query_id "${QUERY_ID}" -q "SELECT * FROM lazy_mark_test WHERE n3==11" +${CLICKHOUSE_CLIENT} --log_queries=1 --query_id "${QUERY_ID}" -q "SELECT * FROM lazy_mark_test WHERE n3==11 SETTINGS load_marks_asynchronously=0" ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS" ${CLICKHOUSE_CLIENT} -q "select ProfileEvents['FileOpen'] from system.query_log where query_id = '${QUERY_ID}' and type = 'QueryFinish' and current_database = currentDatabase()" diff --git a/tests/queries/0_stateless/02267_file_globs_schema_inference.sql b/tests/queries/0_stateless/02267_file_globs_schema_inference.sql index 56b99d80286..6862d6f0602 100644 --- a/tests/queries/0_stateless/02267_file_globs_schema_inference.sql +++ b/tests/queries/0_stateless/02267_file_globs_schema_inference.sql @@ -1,4 +1,5 @@ -- Tags: no-fasttest, no-parallel + insert into function file('02267_data2.jsonl') select NULL as x; insert into function file('02267_data3.jsonl') select * from numbers(0); insert into function file('02267_data4.jsonl') select 1 as x; @@ -7,4 +8,4 @@ select * from file('02267_data*.jsonl') order by x; insert into function file('02267_data1.jsonl', 'TSV') select 1 as x; insert into function file('02267_data1.jsonl', 'TSV') select [1,2,3] as x; -select * from file('02267_data*.jsonl') settings schema_inference_use_cache_for_file=0; --{serverError CANNOT_EXTRACT_TABLE_STRUCTURE} +select * from file('02267_data*.jsonl') settings schema_inference_use_cache_for_file=0; --{serverError INCORRECT_DATA} diff --git a/tests/queries/0_stateless/02268_json_wrong_root_type_in_schema_inference.sql b/tests/queries/0_stateless/02268_json_wrong_root_type_in_schema_inference.sql index 9d435820ce2..7427426602a 100644 --- a/tests/queries/0_stateless/02268_json_wrong_root_type_in_schema_inference.sql +++ b/tests/queries/0_stateless/02268_json_wrong_root_type_in_schema_inference.sql @@ -1,6 +1,7 @@ +-- Tags: no-fasttest + insert into function file('02268_data.jsonl', 'TSV') select 1; -select * from file('02268_data.jsonl'); --{serverError CANNOT_EXTRACT_TABLE_STRUCTURE} +select * from file('02268_data.jsonl'); --{serverError 117} insert into function file('02268_data.jsonCompactEachRow', 'TSV') select 1; -select * from file('02268_data.jsonCompactEachRow'); --{serverError CANNOT_EXTRACT_TABLE_STRUCTURE} - +select * from file('02268_data.jsonCompactEachRow'); --{serverError 117} diff --git a/tests/queries/0_stateless/02286_mysql_dump_input_format.sh b/tests/queries/0_stateless/02286_mysql_dump_input_format.sh index f3253e0bf13..891734e9ad3 100755 --- a/tests/queries/0_stateless/02286_mysql_dump_input_format.sh +++ b/tests/queries/0_stateless/02286_mysql_dump_input_format.sh @@ -23,7 +23,7 @@ $CLICKHOUSE_CLIENT -q "desc file(dump1.sql, MySQLDump) settings input_format_mys $CLICKHOUSE_CLIENT -q "select * from file(dump1.sql, MySQLDump) settings input_format_mysql_dump_table_name='test'" $CLICKHOUSE_CLIENT -q "desc file(dump1.sql, MySQLDump) settings input_format_mysql_dump_table_name='test2'" $CLICKHOUSE_CLIENT -q "select * from file(dump1.sql, MySQLDump) settings input_format_mysql_dump_table_name='test2'" -$CLICKHOUSE_CLIENT -q "desc file(dump1.sql, MySQLDump) settings input_format_mysql_dump_table_name='test 3'" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL' +$CLICKHOUSE_CLIENT -q "desc file(dump1.sql, MySQLDump) settings input_format_mysql_dump_table_name='test 3'" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL' $CLICKHOUSE_CLIENT -q "select * from file(dump1.sql, MySQLDump, 'x Nullable(Int32)') settings input_format_mysql_dump_table_name='test 3'" 2>&1 | grep -F -q 'EMPTY_DATA_PASSED' && echo 'OK' || echo 'FAIL' echo "dump2" @@ -146,4 +146,3 @@ $CLICKHOUSE_CLIENT -q "desc file(dump15.sql, MySQLDump) settings input_format_my $CLICKHOUSE_CLIENT -q "select * from file(dump15.sql, MySQLDump) settings input_format_mysql_dump_table_name='test 3'" rm $USER_FILES_PATH/dump*.sql - diff --git a/tests/queries/0_stateless/02293_formats_json_columns.sh b/tests/queries/0_stateless/02293_formats_json_columns.sh index 74d9a4f5aab..7a21f8d9bab 100755 --- a/tests/queries/0_stateless/02293_formats_json_columns.sh +++ b/tests/queries/0_stateless/02293_formats_json_columns.sh @@ -88,6 +88,4 @@ echo ' } ' > $DATA_FILE -$CLICKHOUSE_CLIENT -q "desc file(data_02293, JSONColumns) settings input_format_max_rows_to_read_for_schema_inference=3" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL' - - +$CLICKHOUSE_CLIENT -q "desc file(data_02293, JSONColumns) settings input_format_max_rows_to_read_for_schema_inference=3" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL' diff --git a/tests/queries/0_stateless/02293_grouping_function.reference b/tests/queries/0_stateless/02293_grouping_function.reference index e71d6812ab5..7d745a0e0fa 100644 --- a/tests/queries/0_stateless/02293_grouping_function.reference +++ b/tests/queries/0_stateless/02293_grouping_function.reference @@ -8,7 +8,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 1 0 1 0 2 @@ -30,7 +31,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 1 0 2 0 2 @@ -52,7 +54,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 1 0 1 @@ -73,7 +76,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, grouping(number, number % 2) = 1; +ORDER BY number, grouping(number, number % 2) = 1 +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 @@ -97,7 +101,8 @@ GROUP BY (number, number % 2), () ) -ORDER BY (gr, number); +ORDER BY (gr, number) +SETTINGS force_grouping_standard_compatibility=0; 0 10 0 0 1 2 1 1 2 @@ -129,7 +134,7 @@ GROUP BY ) HAVING grouping(number, number % 2) = 2 ORDER BY number -SETTINGS enable_optimize_predicate_expression = 0; +SETTINGS enable_optimize_predicate_expression = 0, force_grouping_standard_compatibility=0; 0 1 2 @@ -150,7 +155,7 @@ GROUP BY ) HAVING grouping(number, number % 2) = 1 ORDER BY number -SETTINGS enable_optimize_predicate_expression = 0; +SETTINGS enable_optimize_predicate_expression = 0, force_grouping_standard_compatibility=0; 0 0 SELECT @@ -161,7 +166,8 @@ GROUP BY GROUPING SETS ( (number), (number % 2)) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 1 0 1 diff --git a/tests/queries/0_stateless/02293_grouping_function.sql b/tests/queries/0_stateless/02293_grouping_function.sql index 169fc09c324..cf076c8e51c 100644 --- a/tests/queries/0_stateless/02293_grouping_function.sql +++ b/tests/queries/0_stateless/02293_grouping_function.sql @@ -19,7 +19,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -30,7 +31,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -41,7 +43,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number @@ -51,7 +54,8 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, grouping(number, number % 2) = 1; +ORDER BY number, grouping(number, number % 2) = 1 +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -64,7 +68,8 @@ GROUP BY (number, number % 2), () ) -ORDER BY (gr, number); +ORDER BY (gr, number) +SETTINGS force_grouping_standard_compatibility=0; SELECT number @@ -76,7 +81,7 @@ GROUP BY ) HAVING grouping(number, number % 2) = 2 ORDER BY number -SETTINGS enable_optimize_predicate_expression = 0; +SETTINGS enable_optimize_predicate_expression = 0, force_grouping_standard_compatibility=0; SELECT number @@ -88,7 +93,7 @@ GROUP BY ) HAVING grouping(number, number % 2) = 1 ORDER BY number -SETTINGS enable_optimize_predicate_expression = 0; +SETTINGS enable_optimize_predicate_expression = 0, force_grouping_standard_compatibility=0; SELECT number, @@ -98,4 +103,5 @@ GROUP BY GROUPING SETS ( (number), (number % 2)) -ORDER BY number, gr; +ORDER BY number, gr +SETTINGS force_grouping_standard_compatibility=0; diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.reference b/tests/queries/0_stateless/02293_grouping_function_group_by.reference index 7f87aecd4bd..49cdca1411e 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.reference +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.reference @@ -6,7 +6,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY number, number % 2 -ORDER BY number; +ORDER BY number +SETTINGS force_grouping_standard_compatibility=0; 0 1 1 1 2 1 @@ -25,7 +26,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY number, number % 2 -ORDER BY number; +ORDER BY number +SETTINGS force_grouping_standard_compatibility=0; 0 1 1 1 1 1 2 1 1 @@ -45,7 +47,8 @@ GROUP BY number % 2 WITH ROLLUP ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 2 0 3 @@ -74,7 +77,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY ROLLUP(number, number % 2) ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 2 0 3 @@ -105,7 +109,8 @@ GROUP BY number % 2 WITH CUBE ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 1 0 1 @@ -136,7 +141,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY CUBE(number, number % 2) ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 1 0 1 @@ -168,7 +174,8 @@ GROUP BY CUBE(number, number % 2) HAVING grouping(number) != 0 ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 5 0 6 1 5 @@ -205,7 +212,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY CUBE(number, number % 2) WITH TOTALS ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 1 0 1 @@ -247,7 +255,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY ROLLUP(number, number % 2) WITH TOTALS ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; 0 0 0 2 0 3 diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.sql b/tests/queries/0_stateless/02293_grouping_function_group_by.sql index 9bf9d43478b..d438a8a5277 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.sql +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.sql @@ -15,7 +15,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY number, number % 2 -ORDER BY number; +ORDER BY number +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -25,7 +26,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY number, number % 2 -ORDER BY number; +ORDER BY number +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -36,7 +38,8 @@ GROUP BY number % 2 WITH ROLLUP ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -45,7 +48,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY ROLLUP(number, number % 2) ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -56,7 +60,8 @@ GROUP BY number % 2 WITH CUBE ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -65,7 +70,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY CUBE(number, number % 2) ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -75,7 +81,8 @@ GROUP BY CUBE(number, number % 2) HAVING grouping(number) != 0 ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -94,7 +101,8 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY CUBE(number, number % 2) WITH TOTALS ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; SELECT number, @@ -113,4 +121,5 @@ FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY ROLLUP(number, number % 2) WITH TOTALS ORDER BY - number, gr; + number, gr +SETTINGS force_grouping_standard_compatibility=0; diff --git a/tests/queries/0_stateless/02295_GROUP_BY_AggregateFunction.reference b/tests/queries/0_stateless/02295_GROUP_BY_AggregateFunction.reference index 68355daf334..3489832a25f 100644 --- a/tests/queries/0_stateless/02295_GROUP_BY_AggregateFunction.reference +++ b/tests/queries/0_stateless/02295_GROUP_BY_AggregateFunction.reference @@ -1,14 +1,14 @@ -- { echoOn } -SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg SETTINGS optimize_aggregation_in_order = 0 FORMAT JSONEachRow; -{"grp_aggreg":"\u0002\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} -{"grp_aggreg":"\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} -SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg SETTINGS optimize_aggregation_in_order = 1 FORMAT JSONEachRow; +SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg ORDER BY a SETTINGS optimize_aggregation_in_order = 0 FORMAT JSONEachRow; {"grp_aggreg":"\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} {"grp_aggreg":"\u0002\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} -SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg WITH TOTALS SETTINGS optimize_aggregation_in_order = 0 FORMAT JSONEachRow; -{"grp_aggreg":"\u0002\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} +SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg ORDER BY a SETTINGS optimize_aggregation_in_order = 1 FORMAT JSONEachRow; {"grp_aggreg":"\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} -SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg WITH TOTALS SETTINGS optimize_aggregation_in_order = 1 FORMAT JSONEachRow; +{"grp_aggreg":"\u0002\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} +SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg WITH TOTALS ORDER BY a SETTINGS optimize_aggregation_in_order = 0 FORMAT JSONEachRow; +{"grp_aggreg":"\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} +{"grp_aggreg":"\u0002\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} +SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg WITH TOTALS ORDER BY a SETTINGS optimize_aggregation_in_order = 1 FORMAT JSONEachRow; {"grp_aggreg":"\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} {"grp_aggreg":"\u0002\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0000\u0000\u0000\u0000"} -- regression for incorrect positions passed to finalizeChunk() diff --git a/tests/queries/0_stateless/02295_GROUP_BY_AggregateFunction.sql b/tests/queries/0_stateless/02295_GROUP_BY_AggregateFunction.sql index d4e9369c1c8..862c0f8fde2 100644 --- a/tests/queries/0_stateless/02295_GROUP_BY_AggregateFunction.sql +++ b/tests/queries/0_stateless/02295_GROUP_BY_AggregateFunction.sql @@ -10,10 +10,10 @@ create table data_02295 ( insert into data_02295 select 0 b, intDiv(number, 2) a, groupArrayArrayState([toUInt64(number)]) from numbers(4) group by a, b; -- { echoOn } -SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg SETTINGS optimize_aggregation_in_order = 0 FORMAT JSONEachRow; -SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg SETTINGS optimize_aggregation_in_order = 1 FORMAT JSONEachRow; -SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg WITH TOTALS SETTINGS optimize_aggregation_in_order = 0 FORMAT JSONEachRow; -SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg WITH TOTALS SETTINGS optimize_aggregation_in_order = 1 FORMAT JSONEachRow; +SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg ORDER BY a SETTINGS optimize_aggregation_in_order = 0 FORMAT JSONEachRow; +SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg ORDER BY a SETTINGS optimize_aggregation_in_order = 1 FORMAT JSONEachRow; +SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg WITH TOTALS ORDER BY a SETTINGS optimize_aggregation_in_order = 0 FORMAT JSONEachRow; +SELECT grp_aggreg FROM data_02295 GROUP BY a, grp_aggreg WITH TOTALS ORDER BY a SETTINGS optimize_aggregation_in_order = 1 FORMAT JSONEachRow; -- regression for incorrect positions passed to finalizeChunk() SELECT a, min(b), max(b) FROM data_02295 GROUP BY a ORDER BY a, count() SETTINGS optimize_aggregation_in_order = 1; SELECT a, min(b), max(b) FROM data_02295 GROUP BY a ORDER BY a, count() SETTINGS optimize_aggregation_in_order = 1, max_threads = 1; diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.reference b/tests/queries/0_stateless/02315_grouping_constant_folding.reference index 5aa979b1453..6e591de2661 100644 --- a/tests/queries/0_stateless/02315_grouping_constant_folding.reference +++ b/tests/queries/0_stateless/02315_grouping_constant_folding.reference @@ -1,5 +1,5 @@ -- { echoOn } -SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b); +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0; 1 0 0 3 1 0 2 3 1 0 4 3 @@ -13,7 +13,7 @@ SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING 5 0 0 2 5 1 0 2 10 0 0 0 -SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b); +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0; 1 0 0 3 1 0 2 3 1 0 4 3 diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.sql b/tests/queries/0_stateless/02315_grouping_constant_folding.sql index c4ef087a308..ff259b7be79 100644 --- a/tests/queries/0_stateless/02315_grouping_constant_folding.sql +++ b/tests/queries/0_stateless/02315_grouping_constant_folding.sql @@ -5,9 +5,9 @@ CREATE TABLE test02315(a UInt64, b UInt64) ENGINE=MergeTree() ORDER BY (a, b); INSERT INTO test02315 SELECT number % 2 as a, number as b FROM numbers(10); -- { echoOn } -SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b); +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0; -SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b); +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0; -- { echoOff } DROP TABLE test02315; diff --git a/tests/queries/0_stateless/02327_capnproto_protobuf_empty_messages.sh b/tests/queries/0_stateless/02327_capnproto_protobuf_empty_messages.sh index 9de01dbe294..69e65112305 100755 --- a/tests/queries/0_stateless/02327_capnproto_protobuf_empty_messages.sh +++ b/tests/queries/0_stateless/02327_capnproto_protobuf_empty_messages.sh @@ -15,11 +15,11 @@ mkdir -p $SCHEMADIR/$SERVER_SCHEMADIR cp -r $CLIENT_SCHEMADIR/02327_* $SCHEMADIR/$SERVER_SCHEMADIR/ -$CLICKHOUSE_CLIENT --query="desc file(data.pb) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL'; -$CLICKHOUSE_CLIENT --query="desc file(data.capnp) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="desc file(data.pb) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="desc file(data.capnp) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL'; -$CLICKHOUSE_CLIENT --query="create table test_protobuf engine=File(Protobuf) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL'; -$CLICKHOUSE_CLIENT --query="create table test_capnp engine=File(CapnProto) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="create table test_protobuf engine=File(Protobuf) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="create table test_capnp engine=File(CapnProto) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty'" 2>&1 | grep -F -q 'Cannot extract table structure' && echo 'OK' || echo 'FAIL'; $CLICKHOUSE_CLIENT --query="desc file(data.pb) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty', input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference=1"; $CLICKHOUSE_CLIENT --query="desc file(data.capnp) settings format_schema='$SERVER_SCHEMADIR/02327_schema:MessageWithEmpty', input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference=1"; diff --git a/tests/queries/0_stateless/02337_join_analyze_stuck.sql b/tests/queries/0_stateless/02337_join_analyze_stuck.sql index 62dd0888673..714f43770f5 100644 --- a/tests/queries/0_stateless/02337_join_analyze_stuck.sql +++ b/tests/queries/0_stateless/02337_join_analyze_stuck.sql @@ -2,6 +2,18 @@ -- https://github.com/ClickHouse/ClickHouse/issues/21557 +EXPLAIN SYNTAX +WITH + x AS ( SELECT number FROM numbers(10) ), + cross_sales AS ( + SELECT 1 AS xx + FROM x, x AS d1, x AS d2, x AS d3, x AS d4, x AS d5, x AS d6, x AS d7, x AS d8, x AS d9 + WHERE x.number = d9.number + ) +SELECT xx FROM cross_sales WHERE xx = 2000 FORMAT Null; + +SET max_analyze_depth = 1; + EXPLAIN SYNTAX WITH x AS ( SELECT number FROM numbers(10) ), diff --git a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh index 5079977fea0..e8111bf7a03 100755 --- a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh +++ b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, long CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -16,9 +16,10 @@ function check_refcnt_for_table() query_id="$table-$(random_str 10)" # Notes: - # - query may sleep 0.1*(200/4)=5 seconds, it is enough to check system.parts + # - query may sleep 1*(200/4)=50 seconds maximum, it is enough to check system.parts # - "part = 1" condition should prune all parts except first - $CLICKHOUSE_CLIENT --format Null --max_block_size 1 --query_id "$query_id" -q "select sleepEachRow(0.1) from $table where part = 1" & + # - max_block_size=1 with index_granularity=1 will allow to cancel the query earlier + $CLICKHOUSE_CLIENT --format Null --max_threads 1 --max_block_size 1 --query_id "$query_id" -q "select sleepEachRow(1) from $table where part = 1" & PID=$! # wait for query to be started @@ -30,26 +31,30 @@ function check_refcnt_for_table() # however when it starts reading, partition pruning takes place, # and it should hold only parts that are required for SELECT # - # So 2 seconds delay to ensure that it goes the reading stage. - sleep 2 + # But to reach partition prune the function sleepEachRow() will be executed twice, + # so 2 seconds for sleepEachRow() and 3 seconds just to ensure that it enters the reading stage. + sleep $((2+3)) # NOTE: parts that are used in query will have refcount increased for each range $CLICKHOUSE_CLIENT -q "select table, name, refcount from system.parts where database = '$CLICKHOUSE_DATABASE' and table = '$table' and refcount > 1" + # Kill the query gracefully. kill -INT $PID wait $PID } +# NOTE: index_granularity=1 to cancel ASAP + $CLICKHOUSE_CLIENT -nmq " drop table if exists data_02340; - create table data_02340 (key Int, part Int) engine=MergeTree() partition by part order by key; -" + create table data_02340 (key Int, part Int) engine=MergeTree() partition by part order by key settings index_granularity=1; +" || exit 1 check_refcnt_for_table data_02340 $CLICKHOUSE_CLIENT -nmq " drop table if exists data_02340_rep; - create table data_02340_rep (key Int, part Int) engine=ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') partition by part order by key; -" + create table data_02340_rep (key Int, part Int) engine=ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') partition by part order by key settings index_granularity=1; +" || exit 1 check_refcnt_for_table data_02340_rep exit 0 diff --git a/tests/queries/0_stateless/02366_window_function_order_by.reference b/tests/queries/0_stateless/02366_window_function_order_by.reference new file mode 100644 index 00000000000..89f6c6c911a --- /dev/null +++ b/tests/queries/0_stateless/02366_window_function_order_by.reference @@ -0,0 +1,37 @@ +-- { echoOn } +SELECT groupArray(tuple(value)) OVER () +FROM (select number value from numbers(10)) +ORDER BY value ASC; +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +[(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)] +SELECT count() OVER (ORDER BY number + 1) FROM numbers(10) ORDER BY number; +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +SELECT count() OVER (ORDER BY number + 1) + 1 as foo FROM numbers(10) +ORDER BY foo; +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 diff --git a/tests/queries/0_stateless/02366_window_function_order_by.sql b/tests/queries/0_stateless/02366_window_function_order_by.sql new file mode 100644 index 00000000000..a3a02355cf7 --- /dev/null +++ b/tests/queries/0_stateless/02366_window_function_order_by.sql @@ -0,0 +1,9 @@ +-- { echoOn } +SELECT groupArray(tuple(value)) OVER () +FROM (select number value from numbers(10)) +ORDER BY value ASC; + +SELECT count() OVER (ORDER BY number + 1) FROM numbers(10) ORDER BY number; + +SELECT count() OVER (ORDER BY number + 1) + 1 as foo FROM numbers(10) +ORDER BY foo; diff --git a/tests/queries/0_stateless/02373_datetime64_monotonicity.queries b/tests/queries/0_stateless/02373_datetime64_monotonicity.queries new file mode 100644 index 00000000000..212198c89de --- /dev/null +++ b/tests/queries/0_stateless/02373_datetime64_monotonicity.queries @@ -0,0 +1,57 @@ +drop table if exists dt64_monot_test; +drop table if exists dt64_monot_test_string; +CREATE TABLE dt64_monot_test(`date_time` DateTime64(3, 'Europe/Berlin'), `id` String) ENGINE = MergeTree PARTITION BY toDate(date_time, 'Europe/Berlin') ORDER BY date_time; +insert into dt64_monot_test select toDateTime64('2020-01-01 00:00:00.000',3)+number , '' from numbers(10); + +SELECT count() FROM dt64_monot_test WHERE toDateTime(date_time) >= toDateTime('2020-01-01 00:00:00') SETTINGS force_index_by_date = 1, force_primary_key = 1; + +SELECT count()　FROM dt64_monot_test WHERE toDateTime64(date_time, 3) >= '2020-01-01 00:00:01.111' SETTINGS force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time, 3) >= '2020-01-01 00:00:00.000' SETTINGS force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-01 00:00:00.000001', 3) SETTINGS force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-01 00:00:00.000001', 3, 'Europe/Berlin') SETTINGS force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-01 00:00:00.000001',6) SETTINGS force_index_by_date = 1; -- { serverError 277} + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-01 00:00:00.000001',6, 'Europe/Berlin') SETTINGS force_primary_key = 1; -- { serverError 277} + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) >= toDateTime64('2020-01-01 00:00:00.000001',6) SETTINGS force_primary_key = 1; -- { serverError 277} + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) <= toDateTime64('2020-01-01 00:00:00.000001',3, 'Europe/Berlin') settings force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) <= toDateTime64('2020-01-01 00:00:00.000001',3) settings force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) = toDateTime64('2020-01-01 00:00:00.000000',6); + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,3) = toDateTime64('2020-01-01 00:00:00.000000',6, 'Europe/Berlin'); + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,6) = toDateTime64('2020-01-01 00:00:00.000000',6) settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,6) = toDateTime64('2020-01-01 00:00:00.000001',6, 'Europe/Berlin') settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,6) > toDateTime64('2020-01-01 00:00:00.000001',6, 'Europe/Berlin') settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,6) >= toDateTime64('2020-01-01 00:00:00.000001',6, 'Europe/Berlin') settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,6) >= toDateTime64('2020-01-01 00:00:00.000001',6) settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,0) >= toDateTime64('2020-01-01 00:00:00.000001',0, 'Europe/Berlin') settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,0) >= toDateTime64('2020-01-01 00:00:00.000001',0) settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,0) >= '2020-01-01 00:00:00' settings force_index_by_date = 1, force_primary_key = 1; + +SELECT count() FROM dt64_monot_test WHERE toDateTime64(date_time,0) >= '2020-01-01 00:00:01.1' settings force_index_by_date = 1, force_primary_key = 1; + +create table dt64_monot_test_string(date_time String, x String) Engine=MergeTree order by date_time; +insert into dt64_monot_test_string select '2020-01-01 00:00', '' from numbers(1); +insert into dt64_monot_test_string select '2020-01-01 00:00:00.000000' , '' from numbers(10); + +SELECT count() FROM dt64_monot_test_string WHERE toDateTime64(date_time,3) = '1970-01-01 00:00:00.000000000'; +SELECT count() FROM dt64_monot_test_string WHERE toDateTime64(date_time,3) = '1970-01-01 00:00:00.000000001'; +SELECT count() FROM dt64_monot_test_string WHERE toDateTime64(date_time,9) = '2020-01-01 00:00:00'; + +drop table dt64_monot_test; +drop table dt64_monot_test_string; diff --git a/tests/queries/0_stateless/02373_datetime64_monotonicity.reference b/tests/queries/0_stateless/02373_datetime64_monotonicity.reference new file mode 100644 index 00000000000..d9c310bdbc9 --- /dev/null +++ b/tests/queries/0_stateless/02373_datetime64_monotonicity.reference @@ -0,0 +1,92 @@ +Asia/Tehran +10 +0 +0 +10 +0 +10 +1 +1 +0 +1 +0 +0 +0 +9 +0 +10 +0 +0 +0 +0 +10 + +UTC +10 +10 +10 +10 +10 +0 +1 +1 +0 +1 +0 +10 +10 +9 +10 +10 +10 +10 +1 +1 +10 + +Canada/Atlantic +10 +10 +10 +10 +10 +0 +1 +1 +0 +1 +0 +10 +10 +9 +10 +10 +10 +10 +0 +0 +10 + +Europe/Berlin +10 +8 +10 +10 +10 +1 +1 +1 +1 +1 +0 +9 +9 +9 +10 +10 +10 +9 +0 +0 +10 + diff --git a/tests/queries/0_stateless/02373_datetime64_monotonicity.sh b/tests/queries/0_stateless/02373_datetime64_monotonicity.sh new file mode 100755 index 00000000000..0e0dc0ec22a --- /dev/null +++ b/tests/queries/0_stateless/02373_datetime64_monotonicity.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +for tz in Asia/Tehran UTC Canada/Atlantic Europe/Berlin +do + echo "$tz" + TZ=$tz $CLICKHOUSE_LOCAL -mn < ${CUR_DIR}/02373_datetime64_monotonicity.queries + echo "" +done diff --git a/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql b/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql index 06e1e4bfd55..c3367873042 100644 --- a/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql +++ b/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql @@ -4,6 +4,10 @@ create table m (a int) engine Log; insert into m values (1); +set optimize_rewrite_sum_if_to_count_if=1; + explain syntax select sum(multiIf(a = 1, 1, 0)) from m; +set optimize_rewrite_sum_if_to_count_if=0; + drop table m; diff --git a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference index 1ad64150049..a7498e68bc0 100644 --- a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference +++ b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference @@ -21,7 +21,6 @@ PartialSortingTransform -- ExpressionStep preserves sort mode -- QUERY: set optimize_read_in_order=1;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting ORDER BY a Sorting (Global): a ASC -Sorting Sorting (Global): a ASC Sorting (Stream): a ASC Sorting (Stream): a ASC @@ -66,7 +65,6 @@ Sorting (Global): a ASC Sorting (Sorting for ORDER BY) Sorting (Global): a ASC Sorting (None) -Sorting Sorting (Global): a ASC Sorting (Stream): a ASC Sorting (Stream): a ASC @@ -87,3 +85,9 @@ Sorting (Sorting for ORDER BY) Sorting (Global): plus(a, 1) ASC Sorting (Chunk): a ASC Sorting (Chunk): a ASC +-- check that correct sorting info is provided in case of only prefix of sorting key is in ORDER BY clause but all sorting key columns returned by query +-- QUERY: set optimize_read_in_order=1;EXPLAIN PLAN sorting=1 SELECT a, b FROM optimize_sorting ORDER BY a +Sorting (Global): a ASC +Sorting (Global): a ASC +Sorting (Stream): a ASC +Sorting (Stream): a ASC diff --git a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh index cea76b7d6ea..a308d9bcbc1 100755 --- a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh +++ b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh @@ -8,7 +8,7 @@ DISABLE_OPTIMIZATION="set optimize_sorting_by_input_stream_properties=0;set max_ ENABLE_OPTIMIZATION="set optimize_sorting_by_input_stream_properties=1;set max_threads=1" MAKE_OUTPUT_STABLE="set optimize_read_in_order=1" GREP_SORTING="grep 'PartialSortingTransform\|LimitsCheckingTransform\|MergeSortingTransform\|MergingSortedTransform'" -GREP_SORTMODE="grep 'Sorting'" +GREP_SORTMODE="grep 'Sorting ('" TRIM_LEADING_SPACES="sed -e 's/^[ \t]*//'" FIND_SORTING="$GREP_SORTING | $TRIM_LEADING_SPACES" FIND_SORTMODE="$GREP_SORTMODE | $TRIM_LEADING_SPACES" @@ -72,4 +72,7 @@ explain_sortmode "$MAKE_OUTPUT_STABLE;EXPLAIN PLAN actions=1, header=1, sorting= echo "-- actions chain breaks sorting order: input(column a)->sipHash64(column a)->alias(sipHash64(column a), a)->plus(alias a, 1)" explain_sortmode "$MAKE_OUTPUT_STABLE;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a, z FROM (SELECT sipHash64(a) AS a, a + 1 AS z FROM (SELECT a FROM optimize_sorting ORDER BY a + 1)) ORDER BY a + 1" +echo "-- check that correct sorting info is provided in case of only prefix of sorting key is in ORDER BY clause but all sorting key columns returned by query" +explain_sortmode "$MAKE_OUTPUT_STABLE;EXPLAIN PLAN sorting=1 SELECT a, b FROM optimize_sorting ORDER BY a" + $CLICKHOUSE_CLIENT -q "drop table if exists optimize_sorting sync" diff --git a/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.reference b/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.reference new file mode 100644 index 00000000000..aa950215f59 --- /dev/null +++ b/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.reference @@ -0,0 +1,56 @@ +toStartOfYear;toDate32;true 1920-01-01 +type;toStartOfYear;toDate32;true Date32 +toStartOfYear;toDateTime64;true 1920-01-01 +type;toStartOfYear;toDateTime64;true Date32 +toStartOfISOYear;toDate32;true 1919-12-29 +type;toStartOfISOYear;toDate32;true Date32 +toStartOfISOYear;toDateTime64;true 1919-12-29 +type;toStartOfISOYear;toDateTime64;true Date32 +toStartOfQuarter;toDate32;true 1920-01-01 +type;toStartOfQuarter;toDate32;true Date32 +toStartOfQuarter;toDateTime64;true 1920-01-01 +type;toStartOfQuarter;toDateTime64;true Date32 +toStartOfMonth;toDate32;true 1920-02-01 +type;toStartOfMonth;toDate32;true Date32 +toStartOfMonth;toDateTime64;true 1920-02-01 +type;toStartOfMonth;toDateTime64;true Date32 +toStartOfWeek;toDate32;true 1920-02-01 +type;toStartOfWeek;toDate32;true Date32 +toStartOfWeek;toDateTime64;true 1920-02-01 +type;toStartOfWeek;toDateTime64;true Date32 +toMonday;toDate32;true 1920-02-02 +type;toMonday;toDate32;true Date32 +toMonday;toDateTime64;true 1920-02-02 +type;toMonday;toDateTime64;true Date32 +toLastDayOfMonth;toDate32;true 1920-02-29 +type;toLastDayOfMonth;toDate32;true Date32 +toLastDayOfMonth;toDateTime64;true 1920-02-29 +type;toLastDayOfMonth;toDateTime64;true Date32 +toStartOfYear;toDate32;false 1970-01-01 +type;toStartOfYear;toDate32;false Date +toStartOfYear;toDateTime64;false 1970-01-01 +type;toStartOfYear;toDateTime64;false Date +toStartOfISOYear;toDate32;false 1970-01-01 +type;toStartOfISOYear;toDate32;false Date +toStartOfISOYear;toDateTime64;false 1970-01-01 +type;toStartOfISOYear;toDateTime64;false Date +toStartOfQuarter;toDate32;false 1970-01-01 +type;toStartOfQuarter;toDate32;false Date +toStartOfQuarter;toDateTime64;false 1970-01-01 +type;toStartOfQuarter;toDateTime64;false Date +toStartOfMonth;toDate32;false 1970-01-01 +type;toStartOfMonth;toDate32;false Date +toStartOfMonth;toDateTime64;false 1970-01-01 +type;toStartOfMonth;toDateTime64;false Date +toStartOfWeek;toDate32;false 1970-01-01 +type;toStartOfWeek;toDate32;false Date +toStartOfWeek;toDateTime64;false 1970-01-01 +type;toStartOfWeek;toDateTime64;false Date +toMonday;toDate32;false 1970-01-01 +type;toMonday;toDate32;false Date +toMonday;toDateTime64;false 1970-01-01 +type;toMonday;toDateTime64;false Date +toLastDayOfMonth;toDate32;false 1970-01-01 +type;toLastDayOfMonth;toDate32;false Date +toLastDayOfMonth;toDateTime64;false 1970-01-01 +type;toLastDayOfMonth;toDateTime64;false Date diff --git a/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.sql.j2 b/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.sql.j2 new file mode 100644 index 00000000000..70c07c7792a --- /dev/null +++ b/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.sql.j2 @@ -0,0 +1,9 @@ +{% for option_value in ['true', 'false'] -%} +{% for date_fun in ['toStartOfYear', 'toStartOfISOYear', 'toStartOfQuarter', 'toStartOfMonth', 'toStartOfWeek', 'toMonday', 'toLastDayOfMonth'] -%} +SELECT '{{ date_fun }};toDate32;{{ option_value }}', {{ date_fun }}(toDate32('1920-02-02')) SETTINGS enable_extended_results_for_datetime_functions = {{ option_value }}; +SELECT 'type;{{ date_fun }};toDate32;{{ option_value }}', toTypeName({{ date_fun }}(toDate32('1920-02-02'))) SETTINGS enable_extended_results_for_datetime_functions = {{ option_value }}; +SELECT '{{ date_fun }};toDateTime64;{{ option_value }}', {{ date_fun }}(toDateTime64('1920-02-02 10:20:30', 3)) SETTINGS enable_extended_results_for_datetime_functions = {{ option_value }}; +SELECT 'type;{{ date_fun }};toDateTime64;{{ option_value }}', toTypeName({{ date_fun }}(toDateTime64('1920-02-02 10:20:30', 3))) SETTINGS enable_extended_results_for_datetime_functions = {{ option_value }}; +{% endfor -%} +{% endfor -%} + diff --git a/tests/queries/0_stateless/02413_model_evaluate_smoke.sql b/tests/queries/0_stateless/02413_model_evaluate_smoke.sql deleted file mode 100644 index 3b20067abfe..00000000000 --- a/tests/queries/0_stateless/02413_model_evaluate_smoke.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This model does not exist: -SELECT modelEvaluate('hello', 1, 2, 3); -- { serverError 36 } diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index cbd92d0e8f4..6e0e41f11b8 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -192,6 +192,7 @@ caseWithExpr caseWithExpression caseWithoutExpr caseWithoutExpression +catboostEvaluate cbrt ceil char @@ -475,7 +476,6 @@ min2 minSampleSizeContinous minSampleSizeConversion minus -modelEvaluate modulo moduloLegacy moduloOrZero diff --git a/tests/queries/0_stateless/02416_grouping_function_compatibility.reference b/tests/queries/0_stateless/02416_grouping_function_compatibility.reference new file mode 100644 index 00000000000..c9a3ad2f593 --- /dev/null +++ b/tests/queries/0_stateless/02416_grouping_function_compatibility.reference @@ -0,0 +1,29 @@ +-- { echoOn } +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02416 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b); +1 0 0 0 +1 0 2 0 +1 0 4 0 +1 0 6 0 +1 0 8 0 +1 1 1 0 +1 1 3 0 +1 1 5 0 +1 1 7 0 +1 1 9 0 +5 0 0 1 +5 1 0 1 +10 0 0 3 +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02416 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b); +1 0 0 0 +1 0 2 0 +1 0 4 0 +1 0 6 0 +1 0 8 0 +1 1 1 0 +1 1 3 0 +1 1 5 0 +1 1 7 0 +1 1 9 0 +5 0 0 1 +5 1 0 1 +10 0 0 3 diff --git a/tests/queries/0_stateless/02416_grouping_function_compatibility.sql b/tests/queries/0_stateless/02416_grouping_function_compatibility.sql new file mode 100644 index 00000000000..ed21055ade5 --- /dev/null +++ b/tests/queries/0_stateless/02416_grouping_function_compatibility.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS test02416; + +CREATE TABLE test02416(a UInt64, b UInt64) ENGINE=MergeTree() ORDER BY (a, b); + +INSERT INTO test02416 SELECT number % 2 as a, number as b FROM numbers(10); + +-- { echoOn } +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02416 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b); + +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02416 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b); + +-- { echoOff } +DROP TABLE test02416; + diff --git a/tests/queries/0_stateless/02416_keeper_map.reference b/tests/queries/0_stateless/02416_keeper_map.reference new file mode 100644 index 00000000000..eea8dd975e8 --- /dev/null +++ b/tests/queries/0_stateless/02416_keeper_map.reference @@ -0,0 +1,6 @@ +1 +1 +1 +1 +1 1 1 1 1 +1 diff --git a/tests/queries/0_stateless/02416_keeper_map.sql b/tests/queries/0_stateless/02416_keeper_map.sql new file mode 100644 index 00000000000..c191b539de6 --- /dev/null +++ b/tests/queries/0_stateless/02416_keeper_map.sql @@ -0,0 +1,38 @@ +-- Tags: no-ordinary-database, no-fasttest, long + +DROP TABLE IF EXISTS 02416_test SYNC; + +CREATE TABLE 02416_test (key String, value UInt32) Engine=KeeperMap('/' || currentDatabase() || '/test2416'); -- { serverError 36 } +CREATE TABLE 02416_test (key String, value UInt32) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(key2); -- { serverError 47 } +CREATE TABLE 02416_test (key String, value UInt32) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(key, value); -- { serverError 36 } +CREATE TABLE 02416_test (key String, value UInt32) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(concat(key, value)); -- { serverError 36 } +CREATE TABLE 02416_test (key Tuple(String, UInt32), value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(key); + +DROP TABLE IF EXISTS 02416_test SYNC; +CREATE TABLE 02416_test (key String, value UInt32) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(key); + +INSERT INTO 02416_test SELECT '1_1', number FROM numbers(1000); +SELECT COUNT(1) == 1 FROM 02416_test; + +INSERT INTO 02416_test SELECT concat(toString(number), '_1'), number FROM numbers(1000); +SELECT COUNT(1) == 1000 FROM 02416_test; +SELECT uniqExact(key) == 32 FROM (SELECT * FROM 02416_test LIMIT 32 SETTINGS max_block_size = 1); +SELECT SUM(value) == 1 + 99 + 900 FROM 02416_test WHERE key IN ('1_1', '99_1', '900_1'); + +DROP TABLE IF EXISTS 02416_test SYNC; +DROP TABLE IF EXISTS 02416_test_memory; + +CREATE TABLE 02416_test (k UInt32, value UInt64, dummy Tuple(UInt32, Float64), bm AggregateFunction(groupBitmap, UInt64)) Engine=KeeperMap('/' || currentDatabase() || '/test2416') PRIMARY KEY(k); +CREATE TABLE 02416_test_memory AS 02416_test Engine = Memory; + +INSERT INTO 02416_test SELECT number % 77 AS k, SUM(number) AS value, (1, 1.2), bitmapBuild(groupArray(number)) FROM numbers(10000) group by k; + +INSERT INTO 02416_test_memory SELECT number % 77 AS k, SUM(number) AS value, (1, 1.2), bitmapBuild(groupArray(number)) FROM numbers(10000) group by k; + +SELECT A.a = B.a, A.b = B.b, A.c = B.c, A.d = B.d, A.e = B.e FROM ( SELECT 0 AS a, groupBitmapMerge(bm) AS b , SUM(k) AS c, SUM(value) AS d, SUM(dummy.1) AS e FROM 02416_test) A ANY LEFT JOIN (SELECT 0 AS a, groupBitmapMerge(bm) AS b , SUM(k) AS c, SUM(value) AS d, SUM(dummy.1) AS e FROM 02416_test_memory) B USING a ORDER BY a; + +TRUNCATE TABLE 02416_test; +SELECT 0 == COUNT(1) FROM 02416_test; + +DROP TABLE IF EXISTS 02416_test SYNC; +DROP TABLE IF EXISTS 02416_test_memory; diff --git a/tests/queries/0_stateless/02417_keeper_map_create_drop.reference b/tests/queries/0_stateless/02417_keeper_map_create_drop.reference new file mode 100644 index 00000000000..25bc8c288fb --- /dev/null +++ b/tests/queries/0_stateless/02417_keeper_map_create_drop.reference @@ -0,0 +1,10 @@ +1 11 +------ +1 11 +2 22 +------ +1 11 +2 22 +------ +1 11 +2 22 diff --git a/tests/queries/0_stateless/02417_keeper_map_create_drop.sql b/tests/queries/0_stateless/02417_keeper_map_create_drop.sql new file mode 100644 index 00000000000..49340167eaa --- /dev/null +++ b/tests/queries/0_stateless/02417_keeper_map_create_drop.sql @@ -0,0 +1,20 @@ +-- Tags: no-ordinary-database, no-fasttest + +DROP TABLE IF EXISTS 02417_test SYNC; + +CREATE TABLE 02417_test (key UInt64, value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test2417') PRIMARY KEY(key); +INSERT INTO 02417_test VALUES (1, 11); +SELECT * FROM 02417_test ORDER BY key; +SELECT '------'; + +CREATE TABLE 02417_test_another (key UInt64, value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test2417') PRIMARY KEY(key); +INSERT INTO 02417_test_another VALUES (2, 22); +SELECT * FROM 02417_test_another ORDER BY key; +SELECT '------'; +SELECT * FROM 02417_test ORDER BY key; +SELECT '------'; + +DROP TABLE 02417_test SYNC; +SELECT * FROM 02417_test_another ORDER BY key; + +DROP TABLE 02417_test_another SYNC; diff --git a/tests/queries/0_stateless/02417_load_marks_async.reference b/tests/queries/0_stateless/02417_load_marks_async.reference new file mode 100644 index 00000000000..7326d960397 --- /dev/null +++ b/tests/queries/0_stateless/02417_load_marks_async.reference @@ -0,0 +1 @@ +Ok diff --git a/tests/queries/0_stateless/02417_load_marks_async.sh b/tests/queries/0_stateless/02417_load_marks_async.sh new file mode 100755 index 00000000000..a5cbcd08f75 --- /dev/null +++ b/tests/queries/0_stateless/02417_load_marks_async.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS test;" +${CLICKHOUSE_CLIENT} -n -q " +CREATE TABLE test +( +n0 UInt64, +n1 UInt64, +n2 UInt64, +n3 UInt64, +n4 UInt64, +n5 UInt64, +n6 UInt64, +n7 UInt64, +n8 UInt64, +n9 UInt64 +) +ENGINE = MergeTree +ORDER BY n0 SETTINGS min_bytes_for_wide_part = 1;" + +${CLICKHOUSE_CLIENT} -q "INSERT INTO test select number, number % 3, number % 5, number % 10, number % 13, number % 15, number % 17, number % 18, number % 22, number % 25 from numbers(1000000)" +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES test" + +function test +{ + QUERY_ID=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + + ${CLICKHOUSE_CLIENT} -q "SYSTEM DROP MARK CACHE" + ${CLICKHOUSE_CLIENT} --query_id "${QUERY_ID}" -q "SELECT * FROM test SETTINGS load_marks_asynchronously=$1 FORMAT Null" + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS" + + result=$(${CLICKHOUSE_CLIENT} -q "SELECT ProfileEvents['BackgroundLoadingMarksTasks'] FROM system.query_log WHERE query_id = '${QUERY_ID}' AND type = 'QueryFinish' AND current_database = currentDatabase()") + if [[ $result -ne 0 ]]; then + echo 'Ok' + else + echo 'F' + fi +} + +test 1 diff --git a/tests/queries/0_stateless/02417_null_variadic_behaviour.reference b/tests/queries/0_stateless/02417_null_variadic_behaviour.reference new file mode 100644 index 00000000000..bedb69f99b0 --- /dev/null +++ b/tests/queries/0_stateless/02417_null_variadic_behaviour.reference @@ -0,0 +1,65 @@ +-- { echo } +SELECT avgWeighted(number, number) t, toTypeName(t) FROM numbers(1); +nan Float64 +SELECT avgWeighted(number, number + 1) t, toTypeName(t) FROM numbers(0); +nan Float64 +SELECT avgWeighted(toNullable(number), number) t, toTypeName(t) FROM numbers(1); +nan Nullable(Float64) +SELECT avgWeighted(if(number < 10000, NULL, number), number) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(if(number < 50, NULL, number), number) t, toTypeName(t) FROM numbers(100); +77.29530201342281 Nullable(Float64) +SELECT avgWeighted(number, if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(number, if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +77.29530201342281 Nullable(Float64) +SELECT avgWeighted(toNullable(number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(toNullable(number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +77.29530201342281 Nullable(Float64) +SELECT avgWeighted(if(number < 10000, NULL, number), toNullable(number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(if(number < 50, NULL, number), toNullable(number)) t, toTypeName(t) FROM numbers(100); +77.29530201342281 Nullable(Float64) +SELECT avgWeighted(if(number < 10000, NULL, number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(if(number < 50, NULL, number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(if(number < 10000, NULL, number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeighted(if(number < 50, NULL, number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +77.29530201342281 Nullable(Float64) +SELECT avgWeightedIf(number, number, number % 10) t, toTypeName(t) FROM numbers(100); +66.63333333333334 Float64 +SELECT avgWeightedIf(number, number, toNullable(number % 10)) t, toTypeName(t) FROM numbers(100); +66.63333333333334 Float64 +SELECT avgWeightedIf(number, number, if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +nan Float64 +SELECT avgWeightedIf(number, number, if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +77.75555555555556 Float64 +SELECT avgWeightedIf(number, number, if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +66.63333333333334 Float64 +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +77.75555555555556 Nullable(Float64) +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +\N Nullable(Float64) +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +77.75555555555556 Nullable(Float64) diff --git a/tests/queries/0_stateless/02417_null_variadic_behaviour.sql b/tests/queries/0_stateless/02417_null_variadic_behaviour.sql new file mode 100644 index 00000000000..566cf27bb90 --- /dev/null +++ b/tests/queries/0_stateless/02417_null_variadic_behaviour.sql @@ -0,0 +1,41 @@ +-- { echo } +SELECT avgWeighted(number, number) t, toTypeName(t) FROM numbers(1); +SELECT avgWeighted(number, number + 1) t, toTypeName(t) FROM numbers(0); + +SELECT avgWeighted(toNullable(number), number) t, toTypeName(t) FROM numbers(1); +SELECT avgWeighted(if(number < 10000, NULL, number), number) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 50, NULL, number), number) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeighted(number, if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(number, if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeighted(toNullable(number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(toNullable(number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 10000, NULL, number), toNullable(number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 50, NULL, number), toNullable(number)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeighted(if(number < 10000, NULL, number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 50, NULL, number), if(number < 10000, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 10000, NULL, number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeighted(if(number < 50, NULL, number), if(number < 50, NULL, number)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeightedIf(number, number, number % 10) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(number, number, toNullable(number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(number, number, if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(number, number, if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(number, number, if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 10000, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 50, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); + +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 10000, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 10000, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 10000, NULL, number), if(number < 50, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); +SELECT avgWeightedIf(if(number < 50, NULL, number), if(number < 50, NULL, number), if(number < 0, NULL, number % 10)) t, toTypeName(t) FROM numbers(100); diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference new file mode 100644 index 00000000000..dde07d4540d --- /dev/null +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.reference @@ -0,0 +1,8 @@ +{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} +{"operation_name":"void DB::DistributedSink::writeToLocal(const Cluster::ShardInfo &, const DB::Block &, size_t)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} +{"operation_name":"void DB::StorageDistributedDirectoryMonitor::processFile(const std::string &)","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} +{"operation_name":"void DB::StorageDistributedDirectoryMonitor::processFile(const std::string &)","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} +{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} +{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} +{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"1","rows":"1","bytes":"8"} +{"operation_name":"auto DB::DistributedSink::runWritingJob(DB::DistributedSink::JobReplica &, const DB::Block &, size_t)::(anonymous class)::operator()() const","cluster":"test_cluster_two_shards_localhost","shard":"2","rows":"1","bytes":"8"} diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh new file mode 100755 index 00000000000..9ac5f061d4a --- /dev/null +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, distributed + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +# This function takes 4 arguments: +# $1 - OpenTelemetry Trace Id +# $2 - value of insert_distributed_sync +# $3 - value of prefer_localhost_replica +# $4 - a String that helps to debug +function insert() +{ + echo "INSERT INTO ${CLICKHOUSE_DATABASE}.dist_opentelemetry SETTINGS insert_distributed_sync=$2, prefer_localhost_replica=$3 VALUES(1),(2)" | + ${CLICKHOUSE_CURL} \ + -X POST \ + -H "traceparent: 00-$1-5150000000000515-01" \ + -H "tracestate: $4" \ + "${CLICKHOUSE_URL}" \ + --data @- +} + +function check_span() +{ +${CLICKHOUSE_CLIENT} -nq " + SYSTEM FLUSH LOGS; + + SELECT operation_name, + attribute['clickhouse.cluster'] AS cluster, + attribute['clickhouse.shard_num'] AS shard, + attribute['clickhouse.rows'] AS rows, + attribute['clickhouse.bytes'] AS bytes + FROM system.opentelemetry_span_log + WHERE finish_date >= yesterday() + AND lower(hex(trace_id)) = '${1}' + AND attribute['clickhouse.distributed'] = '${CLICKHOUSE_DATABASE}.dist_opentelemetry' + AND attribute['clickhouse.remote'] = '${CLICKHOUSE_DATABASE}.local_opentelemetry' + ORDER BY attribute['clickhouse.shard_num'] + Format JSONEachRow + ;" +} + + +# +# Prepare tables for tests +# +${CLICKHOUSE_CLIENT} -nq " +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.dist_opentelemetry; +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.local_opentelemetry; + +CREATE TABLE ${CLICKHOUSE_DATABASE}.dist_opentelemetry (key UInt64) Engine=Distributed('test_cluster_two_shards_localhost', ${CLICKHOUSE_DATABASE}, local_opentelemetry, key % 2); +CREATE TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry (key UInt64) Engine=MergeTree ORDER BY key; +" + +# +# test1 +# +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); +insert $trace_id 0 1 "async-insert-writeToLocal" +check_span $trace_id + +# +# test2 +# +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); +insert $trace_id 0 0 "async-insert-writeToRemote" +check_span $trace_id + +# +# test3 +# +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); +insert $trace_id 1 1 "sync-insert-writeToLocal" +check_span $trace_id + +# +# test4 +# +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(generateUUIDv4()))"); +insert $trace_id 1 0 "sync-insert-writeToRemote" +check_span $trace_id + +# +# Cleanup +# +${CLICKHOUSE_CLIENT} -nq " +DROP TABLE ${CLICKHOUSE_DATABASE}.dist_opentelemetry; +DROP TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry; +" diff --git a/tests/queries/0_stateless/02418_aggregate_combinators.reference b/tests/queries/0_stateless/02418_aggregate_combinators.reference new file mode 100644 index 00000000000..499b9862147 --- /dev/null +++ b/tests/queries/0_stateless/02418_aggregate_combinators.reference @@ -0,0 +1,10 @@ +{1:'\0\n\0\0\0\0u,4ƣe\thuULsE|'} +{1:[{1:['\0\n\0\0\0\0u,4ƣe\thuULsE|','\0\n\0\0\0\0u,4ƣe\thuULsE|']}]} +[['\0\n\0\0\0\0u,4ƣe\thuULsE|','\0\nu,4ƣe\th\rH%uULsE|'],[]] +[[{1:'\0\n\0\0\0\0u,4ƣe\thuULsE|'}],[]] +['\0\n\0\0\0\0u,4ƣe\thuULsE|'] +10 +{1:10} +{1:[{1:[10,10]}]} +[[10,10],[]] +[[{1:10}],[]] diff --git a/tests/queries/0_stateless/02418_aggregate_combinators.sql b/tests/queries/0_stateless/02418_aggregate_combinators.sql new file mode 100644 index 00000000000..029660456fd --- /dev/null +++ b/tests/queries/0_stateless/02418_aggregate_combinators.sql @@ -0,0 +1,36 @@ +select uniqStateMap(map(1, number)) from numbers(10); +select uniqStateForEachMapForEachMap(map(1, [map(1, [number, number])])) from numbers(10); +select uniqStateForEachResample(30, 75, 30)([number, number + 1], 30) from numbers(10); +select uniqStateMapForEachResample(30, 75, 30)([map(1, number)], 30) from numbers(10); +select uniqStateForEachMerge(x) as y from (select uniqStateForEachState([number]) as x from numbers(10)); +select uniqMerge(y[1]) from (select uniqStateForEachMerge(x) as y from (select uniqStateForEachState([number]) as x from numbers(10))); + +drop table if exists test; +create table test (x Map(UInt8, AggregateFunction(uniq, UInt64))) engine=Memory; +insert into test select uniqStateMap(map(1, number)) from numbers(10); +select * from test format Null; +select mapApply(k, v -> (k, finalizeAggregation(v)), x) from test; +truncate table test; +drop table test; + +create table test (x Map(UInt8, Array(Map(UInt8, Array(AggregateFunction(uniq, UInt64)))))) engine=Memory; +insert into test select uniqStateForEachMapForEachMap(map(1, [map(1, [number, number])])) from numbers(10); +select mapApply(k, v -> (k, arrayMap(x -> mapApply(k, v -> (k, arrayMap(x -> finalizeAggregation(x), v)), x), v)), x) from test; +select * from test format Null; +truncate table test; +drop table test; + +create table test (x Array(Array(AggregateFunction(uniq, UInt64)))) engine=Memory; +insert into test select uniqStateForEachResample(30, 75, 30)([number, number + 1], 30) from numbers(10); +select arrayMap(x -> arrayMap(x -> finalizeAggregation(x), x), x) from test; +select * from test format Null; +truncate table test; +drop table test; + +create table test (x Array(Array(Map(UInt8, AggregateFunction(uniq, UInt64))))) engine=Memory; +insert into test select uniqStateMapForEachResample(30, 75, 30)([map(1, number)], 30) from numbers(10); +select arrayMap(x -> arrayMap(x -> mapApply(k, v -> (k, finalizeAggregation(v)), x), x), x) from test; +select * from test format Null; +truncate table test; +drop table test; + diff --git a/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.reference b/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.reference new file mode 100644 index 00000000000..f2586c9c42a --- /dev/null +++ b/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.reference @@ -0,0 +1,6 @@ +┌─number─┐ +│ 42 │ +└────────┘ +┌─number─┐ +│ 42 │ +└────────┘ diff --git a/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.sh b/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.sh new file mode 100755 index 00000000000..08c7e18e12c --- /dev/null +++ b/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -ue + +unset CLICKHOUSE_LOG_COMMENT + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CURL} \ + $CLICKHOUSE_URL \ + --get \ + --data-urlencode "query= + select number + from numbers_mt(1e6) + where number = 42 + group by number + settings max_threads = 10, max_bytes_before_external_group_by = 1, group_by_two_level_threshold = 1 + format PrettyCompact" + +${CLICKHOUSE_CURL} \ + $CLICKHOUSE_URL \ + --get \ + --data-urlencode "query= + select number + from numbers_mt(1e6) + where number = 42 + group by number + settings max_threads = 10, max_bytes_before_external_group_by = 0, group_by_two_level_threshold = 1 + format PrettyCompact" diff --git a/tests/queries/0_stateless/02418_keeper_map_keys_limit.reference b/tests/queries/0_stateless/02418_keeper_map_keys_limit.reference new file mode 100644 index 00000000000..95c45d6da51 --- /dev/null +++ b/tests/queries/0_stateless/02418_keeper_map_keys_limit.reference @@ -0,0 +1,4 @@ +2 +3 +4 +4 diff --git a/tests/queries/0_stateless/02418_keeper_map_keys_limit.sql b/tests/queries/0_stateless/02418_keeper_map_keys_limit.sql new file mode 100644 index 00000000000..3d2055b85ea --- /dev/null +++ b/tests/queries/0_stateless/02418_keeper_map_keys_limit.sql @@ -0,0 +1,23 @@ +-- Tags: no-ordinary-database, no-fasttest + +DROP TABLE IF EXISTS 02418_test SYNC; + +CREATE TABLE 02418_test (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 3) PRIMARY KEY(key); + +INSERT INTO 02418_test VALUES (1, 1.1), (2, 2.2); +SELECT count() FROM 02418_test; + +INSERT INTO 02418_test VALUES (3, 3.3), (4, 4.4); -- { serverError 290 } + +INSERT INTO 02418_test VALUES (1, 2.1), (2, 3.2), (3, 3.3); +SELECT count() FROM 02418_test; + +CREATE TABLE 02418_test_another (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 4) PRIMARY KEY(key); +INSERT INTO 02418_test VALUES (4, 4.4); -- { serverError 290 } +INSERT INTO 02418_test_another VALUES (4, 4.4); + +SELECT count() FROM 02418_test; +SELECT count() FROM 02418_test_another; + +DROP TABLE 02418_test SYNC; +DROP TABLE 02418_test_another SYNC; diff --git a/tests/queries/0_stateless/02419_keeper_map_primary_key.reference b/tests/queries/0_stateless/02419_keeper_map_primary_key.reference new file mode 100644 index 00000000000..8394d9f34a7 --- /dev/null +++ b/tests/queries/0_stateless/02419_keeper_map_primary_key.reference @@ -0,0 +1,6 @@ +1.1 +2.2 +1.1 +2.2 +1.1 +2.2 diff --git a/tests/queries/0_stateless/02419_keeper_map_primary_key.sh b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh new file mode 100755 index 00000000000..c43c5bb6408 --- /dev/null +++ b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Tags: no-ordinary-database, no-fasttest, long + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 02419_test SYNC;" + +test_primary_key() +{ + $CLICKHOUSE_CLIENT -nm -q " + CREATE TABLE 02419_test (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 3) PRIMARY KEY($1); + INSERT INTO 02419_test VALUES (1, 1.1), (2, 2.2); + SELECT value FROM 02419_test WHERE key = 1; + SELECT value FROM 02419_test WHERE key IN (2, 3); + DROP TABLE 02419_test SYNC; + " +} + +test_primary_key "sipHash64(key + 42) * 12212121212121" +test_primary_key "reverse(concat(CAST(key, 'String'), 'some string'))" +test_primary_key "hex(toFloat32(key))" diff --git a/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.reference b/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.reference new file mode 100644 index 00000000000..8bd1af11bf2 --- /dev/null +++ b/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.reference @@ -0,0 +1 @@ +2000 diff --git a/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.sql b/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.sql new file mode 100644 index 00000000000..4d2feacfde7 --- /dev/null +++ b/tests/queries/0_stateless/02420_key_condition_actions_dag_bug_40599.sql @@ -0,0 +1,10 @@ +create table tba (event_id Int64, event_dt Int64) Engine =MergeTree order by event_id ; +insert into tba select number%500, 20220822 from numbers(1e6); + +select count() from ( + SELECT event_dt FROM ( + select event_dt, 403 AS event_id from ( + select event_dt from tba as tba + where event_id = 9 and ((tba.event_dt >= 20220822 and tba.event_dt <= 20220822)) + ) + ) tba WHERE tba.event_dt >= 20220822 and tba.event_dt <= 20220822 and event_id = 403 ); diff --git a/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.reference b/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.reference new file mode 100644 index 00000000000..d3d171221e8 --- /dev/null +++ b/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.reference @@ -0,0 +1,10 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql b/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql new file mode 100644 index 00000000000..f5978a34061 --- /dev/null +++ b/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS dtest; + +SELECT count() == 0 FROM (SELECT '33.3' :: Decimal(9, 1) AS a WHERE a IN ('33.33' :: Decimal(9, 2))); + +CREATE TABLE dtest ( `a` Decimal(18, 0), `b` Decimal(18, 1), `c` Decimal(36, 0) ) ENGINE = Memory; +INSERT INTO dtest VALUES ('33', '44.4', '35'); + +SELECT count() == 0 FROM dtest WHERE a IN toDecimal32('33.3000', 4); +SELECT count() == 0 FROM dtest WHERE a IN toDecimal64('33.3000', 4); +SELECT count() == 0 FROM dtest WHERE a IN toDecimal128('33.3000', 4); +SELECT count() == 0 FROM dtest WHERE a IN toDecimal256('33.3000', 4); -- { serverError 53 } + +SELECT count() == 0 FROM dtest WHERE b IN toDecimal32('44.4000', 0); +SELECT count() == 0 FROM dtest WHERE b IN toDecimal64('44.4000', 0); +SELECT count() == 0 FROM dtest WHERE b IN toDecimal128('44.4000', 0); +SELECT count() == 0 FROM dtest WHERE b IN toDecimal256('44.4000', 0); -- { serverError 53 } + +SELECT count() == 1 FROM dtest WHERE b IN toDecimal32('44.4000', 4); +SELECT count() == 1 FROM dtest WHERE b IN toDecimal64('44.4000', 4); +SELECT count() == 1 FROM dtest WHERE b IN toDecimal128('44.4000', 4); +SELECT count() == 1 FROM dtest WHERE b IN toDecimal256('44.4000', 4); -- { serverError 53 } + +DROP TABLE IF EXISTS dtest; diff --git a/tests/queries/0_stateless/02421_exponential_join_rewrite_21557.reference b/tests/queries/0_stateless/02421_exponential_join_rewrite_21557.reference new file mode 100644 index 00000000000..7326d960397 --- /dev/null +++ b/tests/queries/0_stateless/02421_exponential_join_rewrite_21557.reference @@ -0,0 +1 @@ +Ok diff --git a/tests/queries/0_stateless/02421_exponential_join_rewrite_21557.sql b/tests/queries/0_stateless/02421_exponential_join_rewrite_21557.sql new file mode 100644 index 00000000000..f66cd51c1fa --- /dev/null +++ b/tests/queries/0_stateless/02421_exponential_join_rewrite_21557.sql @@ -0,0 +1,452 @@ +-- Tags: long + +-- https://github.com/ClickHouse/ClickHouse/issues/21557 + +DROP TABLE IF EXISTS store_returns; +DROP TABLE IF EXISTS catalog_sales; +DROP TABLE IF EXISTS catalog_returns; +DROP TABLE IF EXISTS date_dim; +DROP TABLE IF EXISTS store; +DROP TABLE IF EXISTS customer; +DROP TABLE IF EXISTS customer_demographics; +DROP TABLE IF EXISTS promotion; +DROP TABLE IF EXISTS household_demographics; +DROP TABLE IF EXISTS customer_address; +DROP TABLE IF EXISTS income_band; +DROP TABLE IF EXISTS item; + +CREATE TABLE store_sales +( + `ss_sold_date_sk` Nullable(Int64), + `ss_sold_time_sk` Nullable(Int64), + `ss_item_sk` Int64, + `ss_customer_sk` Nullable(Int64), + `ss_cdemo_sk` Nullable(Int64), + `ss_hdemo_sk` Nullable(Int64), + `ss_addr_sk` Nullable(Int64), + `ss_store_sk` Nullable(Int64), + `ss_promo_sk` Nullable(Int64), + `ss_ticket_number` Int64, + `ss_quantity` Nullable(Int64), + `ss_wholesale_cost` Nullable(Float32), + `ss_list_price` Nullable(Float32), + `ss_sales_price` Nullable(Float32), + `ss_ext_discount_amt` Nullable(Float32), + `ss_ext_sales_price` Nullable(Float32), + `ss_ext_wholesale_cost` Nullable(Float32), + `ss_ext_list_price` Nullable(Float32), + `ss_ext_tax` Nullable(Float32), + `ss_coupon_amt` Nullable(Float32), + `ss_net_paid` Nullable(Float32), + `ss_net_paid_inc_tax` Nullable(Float32), + `ss_net_profit` Nullable(Float32), + `ss_promo_sk_nn` Int16, + `ss_promo_sk_n2` Nullable(Int16) +) +ENGINE = MergeTree ORDER BY (ss_item_sk, ss_ticket_number); + +CREATE TABLE store_returns +( + `sr_returned_date_sk` Nullable(Int64), + `sr_return_time_sk` Nullable(Int64), + `sr_item_sk` Int64, + `sr_customer_sk` Nullable(Int64), + `sr_cdemo_sk` Nullable(Int64), + `sr_hdemo_sk` Nullable(Int64), + `sr_addr_sk` Nullable(Int64), + `sr_store_sk` Nullable(Int64), + `sr_reason_sk` Nullable(Int64), + `sr_ticket_number` Int64, + `sr_return_quantity` Nullable(Int64), + `sr_return_amt` Nullable(Float32), + `sr_return_tax` Nullable(Float32), + `sr_return_amt_inc_tax` Nullable(Float32), + `sr_fee` Nullable(Float32), + `sr_return_ship_cost` Nullable(Float32), + `sr_refunded_cash` Nullable(Float32), + `sr_reversed_charge` Nullable(Float32), + `sr_store_credit` Nullable(Float32), + `sr_net_loss` Nullable(Float32) +) +ENGINE = MergeTree ORDER BY (sr_item_sk, sr_ticket_number); + +CREATE TABLE catalog_sales +( + `cs_sold_date_sk` Nullable(Int64), + `cs_sold_time_sk` Nullable(Int64), + `cs_ship_date_sk` Nullable(Int64), + `cs_bill_customer_sk` Nullable(Int64), + `cs_bill_cdemo_sk` Nullable(Int64), + `cs_bill_hdemo_sk` Nullable(Int64), + `cs_bill_addr_sk` Nullable(Int64), + `cs_ship_customer_sk` Nullable(Int64), + `cs_ship_cdemo_sk` Nullable(Int64), + `cs_ship_hdemo_sk` Nullable(Int64), + `cs_ship_addr_sk` Nullable(Int64), + `cs_call_center_sk` Nullable(Int64), + `cs_catalog_page_sk` Nullable(Int64), + `cs_ship_mode_sk` Nullable(Int64), + `cs_warehouse_sk` Nullable(Int64), + `cs_item_sk` Int64, + `cs_promo_sk` Nullable(Int64), + `cs_order_number` Int64, + `cs_quantity` Nullable(Int64), + `cs_wholesale_cost` Nullable(Float32), + `cs_list_price` Nullable(Float32), + `cs_sales_price` Nullable(Float32), + `cs_ext_discount_amt` Nullable(Float32), + `cs_ext_sales_price` Nullable(Float32), + `cs_ext_wholesale_cost` Nullable(Float32), + `cs_ext_list_price` Nullable(Float32), + `cs_ext_tax` Nullable(Float32), + `cs_coupon_amt` Nullable(Float32), + `cs_ext_ship_cost` Nullable(Float32), + `cs_net_paid` Nullable(Float32), + `cs_net_paid_inc_tax` Nullable(Float32), + `cs_net_paid_inc_ship` Nullable(Float32), + `cs_net_paid_inc_ship_tax` Nullable(Float32), + `cs_net_profit` Nullable(Float32) +) +ENGINE = MergeTree ORDER BY (cs_item_sk, cs_order_number); + +CREATE TABLE catalog_returns +( + `cr_returned_date_sk` Nullable(Int64), + `cr_returned_time_sk` Nullable(Int64), + `cr_item_sk` Int64, + `cr_refunded_customer_sk` Nullable(Int64), + `cr_refunded_cdemo_sk` Nullable(Int64), + `cr_refunded_hdemo_sk` Nullable(Int64), + `cr_refunded_addr_sk` Nullable(Int64), + `cr_returning_customer_sk` Nullable(Int64), + `cr_returning_cdemo_sk` Nullable(Int64), + `cr_returning_hdemo_sk` Nullable(Int64), + `cr_returning_addr_sk` Nullable(Int64), + `cr_call_center_sk` Nullable(Int64), + `cr_catalog_page_sk` Nullable(Int64), + `cr_ship_mode_sk` Nullable(Int64), + `cr_warehouse_sk` Nullable(Int64), + `cr_reason_sk` Nullable(Int64), + `cr_order_number` Int64, + `cr_return_quantity` Nullable(Int64), + `cr_return_amount` Nullable(Float32), + `cr_return_tax` Nullable(Float32), + `cr_return_amt_inc_tax` Nullable(Float32), + `cr_fee` Nullable(Float32), + `cr_return_ship_cost` Nullable(Float32), + `cr_refunded_cash` Nullable(Float32), + `cr_reversed_charge` Nullable(Float32), + `cr_store_credit` Nullable(Float32), + `cr_net_loss` Nullable(Float32) +) +ENGINE = MergeTree ORDER BY (cr_item_sk, cr_order_number); + +CREATE TABLE date_dim +( + `d_date_sk` Int64, + `d_date_id` String, + `d_date` Nullable(Date), + `d_month_seq` Nullable(Int64), + `d_week_seq` Nullable(Int64), + `d_quarter_seq` Nullable(Int64), + `d_year` Nullable(Int64), + `d_dow` Nullable(Int64), + `d_moy` Nullable(Int64), + `d_dom` Nullable(Int64), + `d_qoy` Nullable(Int64), + `d_fy_year` Nullable(Int64), + `d_fy_quarter_seq` Nullable(Int64), + `d_fy_week_seq` Nullable(Int64), + `d_day_name` Nullable(String), + `d_quarter_name` Nullable(String), + `d_holiday` Nullable(String), + `d_weekend` Nullable(String), + `d_following_holiday` Nullable(String), + `d_first_dom` Nullable(Int64), + `d_last_dom` Nullable(Int64), + `d_same_day_ly` Nullable(Int64), + `d_same_day_lq` Nullable(Int64), + `d_current_day` Nullable(String), + `d_current_week` Nullable(String), + `d_current_month` Nullable(String), + `d_current_quarter` Nullable(String), + `d_current_year` Nullable(String) +) +ENGINE = MergeTree ORDER BY d_date_sk; + +CREATE TABLE store +( + `s_store_sk` Int64, + `s_store_id` String, + `s_rec_start_date` Nullable(Date), + `s_rec_end_date` Nullable(Date), + `s_closed_date_sk` Nullable(Int64), + `s_store_name` Nullable(String), + `s_number_employees` Nullable(Int64), + `s_floor_space` Nullable(Int64), + `s_hours` Nullable(String), + `s_manager` Nullable(String), + `s_market_id` Nullable(Int64), + `s_geography_class` Nullable(String), + `s_market_desc` Nullable(String), + `s_market_manager` Nullable(String), + `s_division_id` Nullable(Int64), + `s_division_name` Nullable(String), + `s_company_id` Nullable(Int64), + `s_company_name` Nullable(String), + `s_street_number` Nullable(String), + `s_street_name` Nullable(String), + `s_street_type` Nullable(String), + `s_suite_number` Nullable(String), + `s_city` Nullable(String), + `s_county` Nullable(String), + `s_state` Nullable(String), + `s_zip` Nullable(String), + `s_country` Nullable(String), + `s_gmt_offset` Nullable(Float32), + `s_tax_precentage` Nullable(Float32) +) +ENGINE = MergeTree ORDER BY s_store_sk; + +CREATE TABLE customer +( + `c_customer_sk` Int64, + `c_customer_id` String, + `c_current_cdemo_sk` Nullable(Int64), + `c_current_hdemo_sk` Nullable(Int64), + `c_current_addr_sk` Nullable(Int64), + `c_first_shipto_date_sk` Nullable(Int64), + `c_first_sales_date_sk` Nullable(Int64), + `c_salutation` Nullable(String), + `c_first_name` Nullable(String), + `c_last_name` Nullable(String), + `c_preferred_cust_flag` Nullable(String), + `c_birth_day` Nullable(Int64), + `c_birth_month` Nullable(Int64), + `c_birth_year` Nullable(Int64), + `c_birth_country` Nullable(String), + `c_login` Nullable(String), + `c_email_address` Nullable(String), + `c_last_review_date` Nullable(String) +) +ENGINE = MergeTree ORDER BY c_customer_sk; + +CREATE TABLE customer_demographics +( + `cd_demo_sk` Int64, + `cd_gender` Nullable(String), + `cd_marital_status` Nullable(String), + `cd_education_status` Nullable(String), + `cd_purchase_estimate` Nullable(Int64), + `cd_credit_rating` Nullable(String), + `cd_dep_count` Nullable(Int64), + `cd_dep_employed_count` Nullable(Int64), + `cd_dep_college_count` Nullable(Int64) +) +ENGINE = MergeTree ORDER BY cd_demo_sk; + +CREATE TABLE promotion +( + `p_promo_sk` Int64, + `p_promo_id` String, + `p_start_date_sk` Nullable(Int64), + `p_end_date_sk` Nullable(Int64), + `p_item_sk` Nullable(Int64), + `p_cost` Nullable(Float64), + `p_response_target` Nullable(Int64), + `p_promo_name` Nullable(String), + `p_channel_dmail` Nullable(String), + `p_channel_email` Nullable(String), + `p_channel_catalog` Nullable(String), + `p_channel_tv` Nullable(String), + `p_channel_radio` Nullable(String), + `p_channel_press` Nullable(String), + `p_channel_event` Nullable(String), + `p_channel_demo` Nullable(String), + `p_channel_details` Nullable(String), + `p_purpose` Nullable(String), + `p_discount_active` Nullable(String) +) +ENGINE = MergeTree ORDER BY p_promo_sk; + +CREATE TABLE household_demographics +( + `hd_demo_sk` Int64, + `hd_income_band_sk` Nullable(Int64), + `hd_buy_potential` Nullable(String), + `hd_dep_count` Nullable(Int64), + `hd_vehicle_count` Nullable(Int64) +) +ENGINE = MergeTree ORDER BY hd_demo_sk; + +CREATE TABLE customer_address +( + `ca_address_sk` Int64, + `ca_address_id` String, + `ca_street_number` Nullable(String), + `ca_street_name` Nullable(String), + `ca_street_type` Nullable(String), + `ca_suite_number` Nullable(String), + `ca_city` Nullable(String), + `ca_county` Nullable(String), + `ca_state` Nullable(String), + `ca_zip` Nullable(String), + `ca_country` Nullable(String), + `ca_gmt_offset` Nullable(Float32), + `ca_location_type` Nullable(String) +) +ENGINE = MergeTree ORDER BY ca_address_sk; + +CREATE TABLE income_band +( + `ib_income_band_sk` Int64, + `ib_lower_bound` Nullable(Int64), + `ib_upper_bound` Nullable(Int64) +) +ENGINE = MergeTree ORDER BY ib_income_band_sk; + +CREATE TABLE item +( + `i_item_sk` Int64, + `i_item_id` String, + `i_rec_start_date` Nullable(Date), + `i_rec_end_date` Nullable(Date), + `i_item_desc` Nullable(String), + `i_current_price` Nullable(Float32), + `i_wholesale_cost` Nullable(Float32), + `i_brand_id` Nullable(Int64), + `i_brand` Nullable(String), + `i_class_id` Nullable(Int64), + `i_class` Nullable(String), + `i_category_id` Nullable(Int64), + `i_category` Nullable(String), + `i_manufact_id` Nullable(Int64), + `i_manufact` Nullable(String), + `i_size` Nullable(String), + `i_formulation` Nullable(String), + `i_color` Nullable(String), + `i_units` Nullable(String), + `i_container` Nullable(String), + `i_manager_id` Nullable(Int64), + `i_product_name` Nullable(String) +) +ENGINE = MergeTree ORDER BY i_item_sk; + +EXPLAIN SYNTAX +WITH + cs_ui AS + ( + SELECT + cs_item_sk, + sum(cs_ext_list_price) AS sale, + sum((cr_refunded_cash + cr_reversed_charge) + cr_store_credit) AS refund + FROM catalog_sales , catalog_returns + WHERE (cs_item_sk = cr_item_sk) AND (cs_order_number = cr_order_number) + GROUP BY cs_item_sk + HAVING sum(cs_ext_list_price) > (2 * sum((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)) + ), + cross_sales AS + ( + SELECT + i_product_name AS product_name, + i_item_sk AS item_sk, + s_store_name AS store_name, + s_zip AS store_zip, + ad1.ca_street_number AS b_street_number, + ad1.ca_street_name AS b_street_name, + ad1.ca_city AS b_city, + ad1.ca_zip AS b_zip, + ad2.ca_street_number AS c_street_number, + ad2.ca_street_name AS c_street_name, + ad2.ca_city AS c_city, + ad2.ca_zip AS c_zip, + d1.d_year AS syear, + d2.d_year AS fsyear, + d3.d_year AS s2year, + count(*) AS cnt, + sum(ss_wholesale_cost) AS s1, + sum(ss_list_price) AS s2, + sum(ss_coupon_amt) AS s3 + FROM store_sales + , store_returns + , cs_ui + , date_dim AS d1 + , date_dim AS d2 + , date_dim AS d3 + , store + , customer + , customer_demographics AS cd1 + , customer_demographics AS cd2 + , promotion + , household_demographics AS hd1 + , household_demographics AS hd2 + , customer_address AS ad1 + , customer_address AS ad2 + , income_band AS ib1 + , income_band AS ib2 + , item + WHERE (ss_store_sk = s_store_sk) AND (ss_sold_date_sk = d1.d_date_sk) AND (ss_customer_sk = c_customer_sk) AND (ss_cdemo_sk = cd1.cd_demo_sk) AND (ss_hdemo_sk = hd1.hd_demo_sk) AND (ss_addr_sk = ad1.ca_address_sk) AND (ss_item_sk = i_item_sk) AND (ss_item_sk = sr_item_sk) AND (ss_ticket_number = sr_ticket_number) AND (ss_item_sk = cs_ui.cs_item_sk) AND (c_current_cdemo_sk = cd2.cd_demo_sk) AND (c_current_hdemo_sk = hd2.hd_demo_sk) AND (c_current_addr_sk = ad2.ca_address_sk) AND (c_first_sales_date_sk = d2.d_date_sk) AND (c_first_shipto_date_sk = d3.d_date_sk) AND (ss_promo_sk = p_promo_sk) AND (hd1.hd_income_band_sk = ib1.ib_income_band_sk) AND (hd2.hd_income_band_sk = ib2.ib_income_band_sk) AND (cd1.cd_marital_status != cd2.cd_marital_status) AND (i_color IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate')) AND ((i_current_price >= 35) AND (i_current_price <= (35 + 10))) AND ((i_current_price >= (35 + 1)) AND (i_current_price <= (35 + 15))) + GROUP BY + i_product_name, + i_item_sk, + s_store_name, + s_zip, + ad1.ca_street_number, + ad1.ca_street_name, + ad1.ca_city, + ad1.ca_zip, + ad2.ca_street_number, + ad2.ca_street_name, + ad2.ca_city, + ad2.ca_zip, + d1.d_year, + d2.d_year, + d3.d_year + ) +SELECT + cs1.product_name, + cs1.store_name, + cs1.store_zip, + cs1.b_street_number, + cs1.b_street_name, + cs1.b_city, + cs1.b_zip, + cs1.c_street_number, + cs1.c_street_name, + cs1.c_city, + cs1.c_zip, + cs1.syear, + cs1.cnt, + cs1.s1 AS s11, + cs1.s2 AS s21, + cs1.s3 AS s31, + cs2.s1 AS s12, + cs2.s2 AS s22, + cs2.s3 AS s32, + cs2.syear, + cs2.cnt +FROM cross_sales AS cs1 , cross_sales AS cs2 +WHERE (cs1.item_sk = cs2.item_sk) AND (cs1.syear = 2000) AND (cs2.syear = (2000 + 1)) AND (cs2.cnt <= cs1.cnt) AND (cs1.store_name = cs2.store_name) AND (cs1.store_zip = cs2.store_zip) +ORDER BY + cs1.product_name ASC, + cs1.store_name ASC, + cs2.cnt ASC, + cs1.s1 ASC, + cs2.s1 ASC +FORMAT Null +; + +SELECT 'Ok'; + +DROP TABLE IF EXISTS store_returns; +DROP TABLE IF EXISTS catalog_sales; +DROP TABLE IF EXISTS catalog_returns; +DROP TABLE IF EXISTS date_dim; +DROP TABLE IF EXISTS store; +DROP TABLE IF EXISTS customer; +DROP TABLE IF EXISTS customer_demographics; +DROP TABLE IF EXISTS promotion; +DROP TABLE IF EXISTS household_demographics; +DROP TABLE IF EXISTS customer_address; +DROP TABLE IF EXISTS income_band; +DROP TABLE IF EXISTS item; diff --git a/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.reference b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.reference new file mode 100644 index 00000000000..ee8e589089c Binary files /dev/null and b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.reference differ diff --git a/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 new file mode 100644 index 00000000000..32738766199 --- /dev/null +++ b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 @@ -0,0 +1,22 @@ +-- Tags: no-fasttest + +set output_format_write_statistics=0; + +{% for format in ['CSV', 'TSV', 'XML', 'Vertical', 'Pretty', 'JSON', 'JSONCompact'] -%} + +select '{{ format }}'; +select sum(number) from numbers(10) group by number % 2 with totals format {{ format }} settings extremes=1; +select ''; + +{% endfor -%} + +select 'Formats without totals and extremes:'; + +{% for format in ['CustomSeparated', 'JSONEachRow', 'JSONCompactEachRow', 'RowBinary', 'MsgPack', 'Markdown', 'SQLInsert', 'Values', 'TSKV'] -%} + +select '{{ format }}'; +select sum(number) from numbers(10) group by number % 2 with totals format {{ format }} settings extremes=1; +select ''; + +{% endfor -%} + diff --git a/tests/queries/0_stateless/02421_record_errors_row_by_input_format.reference b/tests/queries/0_stateless/02421_record_errors_row_by_input_format.reference new file mode 100644 index 00000000000..67ec09b70b7 --- /dev/null +++ b/tests/queries/0_stateless/02421_record_errors_row_by_input_format.reference @@ -0,0 +1,6 @@ +default data 2 Row 2:\nColumn 0, name: c1, type: UInt8, parsed text: "2"\nColumn 1, name: c2, type: UInt8, ERROR: text "ab,34,4" is not like UInt8 2,a +default data 3 Row 3:\nColumn 0, name: c1, type: UInt8, ERROR: text "b,34,45," is not like UInt8 b,3 +default data 5 Row 5:\nColumn 0, name: c1, type: UInt8, parsed text: "5"\nColumn 1, name: c2, type: UInt8, ERROR: text "c6,6" is not like UInt8 5,c +\N data 2 Row 2:\nColumn 0, name: A, type: UInt8, parsed text: "2"\nColumn 1, name: B, type: UInt8, ERROR: text "ab,34,4" is not like UInt8 2,a +\N data 3 Row 3:\nColumn 0, name: A, type: UInt8, ERROR: text "b,34,45," is not like UInt8 b,3 +\N data 5 Row 5:\nColumn 0, name: A, type: UInt8, parsed text: "5"\nColumn 1, name: B, type: UInt8, ERROR: text "c6,6" is not like UInt8 5,c diff --git a/tests/queries/0_stateless/02421_record_errors_row_by_input_format.sh b/tests/queries/0_stateless/02421_record_errors_row_by_input_format.sh new file mode 100755 index 00000000000..dda61512936 --- /dev/null +++ b/tests/queries/0_stateless/02421_record_errors_row_by_input_format.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Tags: no-parallel, no-fasttest + +set -eu + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Data preparation. + +CLICKHOUSE_USER_FILES_PATH=$(clickhouse-client --query "select _path, _file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +mkdir -p ${CLICKHOUSE_USER_FILES_PATH}/ +echo -e "1,1\n2,a\nb,3\n4,4\n5,c\n6,6" > ${CLICKHOUSE_USER_FILES_PATH}/a.csv + +${CLICKHOUSE_CLIENT} --query "drop table if exists data;" +${CLICKHOUSE_CLIENT} --query "create table data (A UInt8, B UInt8) engine=MergeTree() order by A;" + +# Server side +${CLICKHOUSE_CLIENT} --input_format_allow_errors_num 4 --input_format_record_errors_file_path "errors_server" --query "insert into data select * from file('a.csv', 'CSV', 'c1 UInt8, c2 UInt8');" +sleep 2 +${CLICKHOUSE_CLIENT} --query "select * except (time) from file('errors_server', 'CSV', 'time DateTime, database Nullable(String), table Nullable(String), offset UInt32, reason String, raw_data String');" + +# Client side +${CLICKHOUSE_CLIENT} --input_format_allow_errors_num 4 --input_format_record_errors_file_path "${CLICKHOUSE_USER_FILES_PATH}/errors_client" --query "insert into data(A, B) format CSV" < ${CLICKHOUSE_USER_FILES_PATH}/a.csv +sleep 2 +${CLICKHOUSE_CLIENT} --query "select * except (time) from file('errors_client', 'CSV', 'time DateTime, database Nullable(String), table Nullable(String), offset UInt32, reason String, raw_data String');" + +# Restore +${CLICKHOUSE_CLIENT} --query "drop table if exists data;" +rm ${CLICKHOUSE_USER_FILES_PATH}/a.csv +rm ${CLICKHOUSE_USER_FILES_PATH}/errors_server +rm ${CLICKHOUSE_USER_FILES_PATH}/errors_client + diff --git a/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.reference b/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.reference new file mode 100644 index 00000000000..d167d905636 --- /dev/null +++ b/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.reference @@ -0,0 +1,4 @@ +{"query":"show processlist format Null\n "} +{"query":"show databases format Null\n "} +{"query":"insert into opentelemetry_test values","read_rows":"3","written_rows":"3"} +{"query":"select * from opentelemetry_test format Null\n ","read_rows":"3","written_rows":""} diff --git a/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh b/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh new file mode 100755 index 00000000000..98b571c5968 --- /dev/null +++ b/tests/queries/0_stateless/02421_simple_queries_for_opentelemetry.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# This function takes 2 arguments: +# $1 - query id +# $2 - query +function execute_query() +{ + ${CLICKHOUSE_CLIENT} --opentelemetry_start_trace_probability=1 --query_id $1 -nq " + ${2} + " +} + +# For some queries, it's not possible to know how many bytes/rows are read when tests are executed on CI, +# so we only to check the db.statement only +function check_query_span_query_only() +{ +${CLICKHOUSE_CLIENT} -nq " + SYSTEM FLUSH LOGS; + SELECT attribute['db.statement'] as query + FROM system.opentelemetry_span_log + WHERE finish_date >= yesterday() + AND operation_name = 'query' + AND attribute['clickhouse.query_id'] = '${1}' + Format JSONEachRow + ;" +} + +function check_query_span() +{ +${CLICKHOUSE_CLIENT} -nq " + SYSTEM FLUSH LOGS; + SELECT attribute['db.statement'] as query, + attribute['clickhouse.read_rows'] as read_rows, + attribute['clickhouse.written_rows'] as written_rows + FROM system.opentelemetry_span_log + WHERE finish_date >= yesterday() + AND operation_name = 'query' + AND attribute['clickhouse.query_id'] = '${1}' + Format JSONEachRow + ;" +} + +# +# Set up +# +${CLICKHOUSE_CLIENT} -nq " +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.opentelemetry_test; +CREATE TABLE ${CLICKHOUSE_DATABASE}.opentelemetry_test (id UInt64) Engine=MergeTree Order By id; +" + +# test 1, a query that has special path in the code +# Format Null is used to make sure no output is generated so that it won't pollute the reference file +query_id=$(${CLICKHOUSE_CLIENT} -q "select generateUUIDv4()"); +execute_query $query_id 'show processlist format Null' +check_query_span_query_only "$query_id" + +# test 2, a normal show command +query_id=$(${CLICKHOUSE_CLIENT} -q "select generateUUIDv4()"); +execute_query $query_id 'show databases format Null' +check_query_span_query_only "$query_id" + +# test 3, a normal insert query on local table +query_id=$(${CLICKHOUSE_CLIENT} -q "select generateUUIDv4()"); +execute_query $query_id 'insert into opentelemetry_test values(1)(2)(3)' +check_query_span "$query_id" + +# test 4, a normal select query +query_id=$(${CLICKHOUSE_CLIENT} -q "select generateUUIDv4()"); +execute_query $query_id 'select * from opentelemetry_test format Null' +check_query_span $query_id + + +# +# Tear down +# +${CLICKHOUSE_CLIENT} -q " +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.opentelemetry_test; +" \ No newline at end of file diff --git a/tests/queries/0_stateless/02421_type_json_async_insert.reference b/tests/queries/0_stateless/02421_type_json_async_insert.reference new file mode 100644 index 00000000000..f3d96ebf2d0 --- /dev/null +++ b/tests/queries/0_stateless/02421_type_json_async_insert.reference @@ -0,0 +1,5 @@ +Cannot parse object +0 +0 +Cannot parse object +aaa diff --git a/tests/queries/0_stateless/02421_type_json_async_insert.sh b/tests/queries/0_stateless/02421_type_json_async_insert.sh new file mode 100755 index 00000000000..8aa0d510dbb --- /dev/null +++ b/tests/queries/0_stateless/02421_type_json_async_insert.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_async_insert" +$CLICKHOUSE_CLIENT --allow_experimental_object_type=1 -q "CREATE TABLE t_json_async_insert (data JSON) ENGINE = MergeTree ORDER BY tuple()" + +$CLICKHOUSE_CLIENT --async_insert=1 --wait_for_async_insert=1 -q 'INSERT INTO t_json_async_insert FORMAT JSONAsObject {"aaa"}' 2>&1 | grep -o -m1 "Cannot parse object" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM t_json_async_insert" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 't_json_async_insert'" + +$CLICKHOUSE_CLIENT --async_insert=1 --wait_for_async_insert=1 -q 'INSERT INTO t_json_async_insert FORMAT JSONAsObject {"aaa"}' 2>&1 | grep -o -m1 "Cannot parse object" & +$CLICKHOUSE_CLIENT --async_insert=1 --wait_for_async_insert=1 -q 'INSERT INTO t_json_async_insert FORMAT JSONAsObject {"k1": "aaa"}' & + +wait + +$CLICKHOUSE_CLIENT -q "SELECT data.k1 FROM t_json_async_insert ORDER BY data.k1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_async_insert" diff --git a/tests/queries/0_stateless/02421_type_json_empty_parts.reference b/tests/queries/0_stateless/02421_type_json_empty_parts.reference new file mode 100644 index 00000000000..f360b4b92cd --- /dev/null +++ b/tests/queries/0_stateless/02421_type_json_empty_parts.reference @@ -0,0 +1,26 @@ +Collapsing +0 +0 +id UInt64 +s Int8 +data Tuple(_dummy UInt8) +DELETE all +2 +1 +id UInt64 +data Tuple(k1 String, k2 String) +0 +0 +id UInt64 +data Tuple(_dummy UInt8) +TTL +1 +1 +id UInt64 +d Date +data Tuple(k1 String, k2 String) +0 +0 +id UInt64 +d Date +data Tuple(_dummy UInt8) diff --git a/tests/queries/0_stateless/02421_type_json_empty_parts.sql b/tests/queries/0_stateless/02421_type_json_empty_parts.sql new file mode 100644 index 00000000000..409a2b18a49 --- /dev/null +++ b/tests/queries/0_stateless/02421_type_json_empty_parts.sql @@ -0,0 +1,61 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + +DROP TABLE IF EXISTS t_json_empty_parts; + +SELECT 'Collapsing'; +CREATE TABLE t_json_empty_parts (id UInt64, s Int8, data JSON) ENGINE = CollapsingMergeTree(s) ORDER BY id; + +INSERT INTO t_json_empty_parts VALUES (1, 1, '{"k1": "aaa"}') (1, -1, '{"k2": "bbb"}'); + +SELECT count() FROM t_json_empty_parts; +SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active; +DESC TABLE t_json_empty_parts SETTINGS describe_extend_object_types = 1; + +DROP TABLE t_json_empty_parts; + +DROP TABLE IF EXISTS t_json_empty_parts; + +SELECT 'DELETE all'; +CREATE TABLE t_json_empty_parts (id UInt64, data JSON) ENGINE = MergeTree ORDER BY id; + +INSERT INTO t_json_empty_parts VALUES (1, '{"k1": "aaa"}') (2, '{"k2": "bbb"}'); + +SELECT count() FROM t_json_empty_parts; +SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active; +DESC TABLE t_json_empty_parts SETTINGS describe_extend_object_types = 1; + +SET mutations_sync = 2; +ALTER TABLE t_json_empty_parts DELETE WHERE 1; + +DETACH TABLE t_json_empty_parts; +ATTACH TABLE t_json_empty_parts; + +SELECT count() FROM t_json_empty_parts; +SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active; +DESC TABLE t_json_empty_parts SETTINGS describe_extend_object_types = 1; + +DROP TABLE IF EXISTS t_json_empty_parts; + +SELECT 'TTL'; +CREATE TABLE t_json_empty_parts (id UInt64, d Date, data JSON) ENGINE = MergeTree ORDER BY id TTL d WHERE id % 2 = 1; + +INSERT INTO t_json_empty_parts VALUES (1, '2000-01-01', '{"k1": "aaa"}') (2, '2000-01-01', '{"k2": "bbb"}'); +OPTIMIZE TABLE t_json_empty_parts FINAL; + +SELECT count() FROM t_json_empty_parts; +SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active; +DESC TABLE t_json_empty_parts SETTINGS describe_extend_object_types = 1; + +ALTER TABLE t_json_empty_parts MODIFY TTL d; +OPTIMIZE TABLE t_json_empty_parts FINAL; + +DETACH TABLE t_json_empty_parts; +ATTACH TABLE t_json_empty_parts; + +SELECT count() FROM t_json_empty_parts; +SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active; +DESC TABLE t_json_empty_parts SETTINGS describe_extend_object_types = 1; + +DROP TABLE IF EXISTS t_json_empty_parts; diff --git a/tests/queries/0_stateless/02413_model_evaluate_smoke.reference b/tests/queries/0_stateless/02422_msgpack_uuid_wrong_column.reference similarity index 100% rename from tests/queries/0_stateless/02413_model_evaluate_smoke.reference rename to tests/queries/0_stateless/02422_msgpack_uuid_wrong_column.reference diff --git a/tests/queries/0_stateless/02422_msgpack_uuid_wrong_column.sql b/tests/queries/0_stateless/02422_msgpack_uuid_wrong_column.sql new file mode 100644 index 00000000000..4d790354d51 --- /dev/null +++ b/tests/queries/0_stateless/02422_msgpack_uuid_wrong_column.sql @@ -0,0 +1,4 @@ +-- Tags: no-parallel, no-fasttest + +insert into function file(02422_data.msgpack) select toUUID('f4cdd80d-5d15-4bdc-9527-adcca635ec1f') as uuid settings output_format_msgpack_uuid_representation='ext'; +select * from file(02422_data.msgpack, auto, 'x Int32'); -- {serverError ILLEGAL_COLUMN} diff --git a/tests/queries/0_stateless/02423_insert_stats_behaviour.reference b/tests/queries/0_stateless/02423_insert_stats_behaviour.reference new file mode 100644 index 00000000000..6aabb57a8bd --- /dev/null +++ b/tests/queries/0_stateless/02423_insert_stats_behaviour.reference @@ -0,0 +1,12 @@ +read_rows=1 read_bytes=8 written_rows=1 written_bytes=8 query=INSERT into target_1 FORMAT CSV +read_rows=10 read_bytes=80 written_rows=10 written_bytes=80 query=INSERT INTO target_1 FORMAT Native\n +read_rows=10 read_bytes=80 written_rows=10 written_bytes=80 query=INSERT INTO target_1 FORMAT RowBinary\n +read_rows=5 read_bytes=40 written_rows=4 written_bytes=32 query=INSERT into floats FORMAT CSV +read_rows=32 read_bytes=256 written_rows=40 written_bytes=320 query=INSERT INTO floats FORMAT Native\n +read_rows=32 read_bytes=256 written_rows=40 written_bytes=320 query=INSERT INTO floats FORMAT RowBinary\n +read_rows=1 read_bytes=8 written_rows=1 written_bytes=8 source_query=INSERT into floats FORMAT CSV view_query=SELECT * FROM default.floats +read_rows=3 read_bytes=24 written_rows=2 written_bytes=16 source_query=INSERT into floats FORMAT CSV view_query=SELECT * FROM default.floats, numbers(2) AS n +read_rows=10 read_bytes=80 written_rows=10 written_bytes=80 source_query=INSERT INTO floats FORMAT Native\n view_query=SELECT * FROM default.floats +read_rows=12 read_bytes=96 written_rows=20 written_bytes=160 source_query=INSERT INTO floats FORMAT Native\n view_query=SELECT * FROM default.floats, numbers(2) AS n +read_rows=10 read_bytes=80 written_rows=10 written_bytes=80 source_query=INSERT INTO floats FORMAT RowBinary\n view_query=SELECT * FROM default.floats +read_rows=12 read_bytes=96 written_rows=20 written_bytes=160 source_query=INSERT INTO floats FORMAT RowBinary\n view_query=SELECT * FROM default.floats, numbers(2) AS n diff --git a/tests/queries/0_stateless/02423_insert_stats_behaviour.sh b/tests/queries/0_stateless/02423_insert_stats_behaviour.sh new file mode 100755 index 00000000000..b85ca311101 --- /dev/null +++ b/tests/queries/0_stateless/02423_insert_stats_behaviour.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "CREATE TABLE floats (v Float64) Engine=MergeTree() ORDER BY tuple();" +$CLICKHOUSE_CLIENT -q "CREATE TABLE target_1 (v Float64) Engine=MergeTree() ORDER BY tuple();" +$CLICKHOUSE_CLIENT -q "CREATE TABLE target_2 (v Float64) Engine=MergeTree() ORDER BY tuple();" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW floats_to_target TO target_1 AS SELECT * FROM floats" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW floats_to_target_2 TO target_2 AS SELECT * FROM floats, numbers(2) n" + +# Insertions into table without MVs +$CLICKHOUSE_CLIENT -q "INSERT into target_1 FORMAT CSV 1.0" +$CLICKHOUSE_LOCAL -q "SELECT number::Float64 AS v FROM numbers(10)" --format Native | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+target_1+FORMAT+Native" --data-binary @- +$CLICKHOUSE_LOCAL -q "SELECT number::Float64 AS v FROM numbers(10)" --format RowBinary | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+target_1+FORMAT+RowBinary" --data-binary @- + +# Insertions into table without 2 MVs (1:1 and 1:2 rows) +$CLICKHOUSE_CLIENT -q "INSERT into floats FORMAT CSV 1.0" +$CLICKHOUSE_LOCAL -q "SELECT number::Float64 AS v FROM numbers(10)" --format Native | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+floats+FORMAT+Native" --data-binary @- +$CLICKHOUSE_LOCAL -q "SELECT number::Float64 AS v FROM numbers(10)" --format RowBinary | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+floats+FORMAT+RowBinary" --data-binary @- + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS" +$CLICKHOUSE_CLIENT -q \ + "SELECT + read_rows, + read_bytes, + written_rows, + written_bytes, + query + FROM system.query_log + WHERE + event_date >= yesterday() + AND event_time > now() - INTERVAL 600 SECOND + AND type = 'QueryFinish' + AND query_kind = 'Insert' + AND current_database == currentDatabase() + ORDER BY event_time_microseconds ASC + FORMAT TSKV" + +$CLICKHOUSE_CLIENT -q \ + "SELECT + read_rows, + read_bytes, + written_rows, + written_bytes, + ql.query as source_query, + view_query + FROM system.query_views_log + INNER JOIN + ( + SELECT + query_id, query, event_time_microseconds + FROM system.query_log + WHERE + event_date >= yesterday() + AND event_time > now() - INTERVAL 600 SECOND + AND type = 'QueryFinish' + AND query_kind = 'Insert' + AND current_database == currentDatabase() + ) ql + ON system.query_views_log.initial_query_id = ql.query_id + ORDER BY ql.event_time_microseconds ASC, view_query ASC + FORMAT TSKV" diff --git a/tests/queries/0_stateless/02423_multidimensional_array_get_data_at.reference b/tests/queries/0_stateless/02423_multidimensional_array_get_data_at.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02423_multidimensional_array_get_data_at.sql b/tests/queries/0_stateless/02423_multidimensional_array_get_data_at.sql new file mode 100644 index 00000000000..a47fbdfc789 --- /dev/null +++ b/tests/queries/0_stateless/02423_multidimensional_array_get_data_at.sql @@ -0,0 +1,7 @@ +SELECT formatRow('RawBLOB', [[[33]], []]); -- { serverError 48 } +SELECT formatRow('RawBLOB', [[[]], []]); -- { serverError 48 } +SELECT formatRow('RawBLOB', [[[[[[[0x48, 0x65, 0x6c, 0x6c, 0x6f]]]]]], []]); -- { serverError 48 } +SELECT formatRow('RawBLOB', []::Array(Array(Nothing))); -- { serverError 48 } +SELECT formatRow('RawBLOB', [[], [['Hello']]]); -- { serverError 48 } +SELECT formatRow('RawBLOB', [[['World']], []]); -- { serverError 48 } +SELECT formatRow('RawBLOB', []::Array(String)); -- { serverError 48 } diff --git a/tests/queries/0_stateless/02424_pod_array_overflow.reference b/tests/queries/0_stateless/02424_pod_array_overflow.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02424_pod_array_overflow.sql b/tests/queries/0_stateless/02424_pod_array_overflow.sql new file mode 100644 index 00000000000..4b85d5be029 --- /dev/null +++ b/tests/queries/0_stateless/02424_pod_array_overflow.sql @@ -0,0 +1 @@ +SELECT * FROM format(Native, '\x02\x02\x02\x6b\x30\x1a\x4d\x61\x70\x28\x46\x69\x78\x65\x64\x53\x74\x72\x69\x6e\x67\x28\x31\x29\x2c\x20\x49\x6e\x74\x36\x34\x29\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x7f\x00\x7f\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x64\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xcf\x31\x3f\x56\x69\x11\x89\x25'); -- { serverError 128 } diff --git a/tests/queries/0_stateless/02425_categorical_information_value_properties.reference b/tests/queries/0_stateless/02425_categorical_information_value_properties.reference new file mode 100644 index 00000000000..bc3af98b060 --- /dev/null +++ b/tests/queries/0_stateless/02425_categorical_information_value_properties.reference @@ -0,0 +1,12 @@ +0.347 +0.5 +0.347 +0.347 +[nan] +[nan] +[nan] +[nan] +[0] +\N +[nan] +[0,0] diff --git a/tests/queries/0_stateless/02425_categorical_information_value_properties.sql b/tests/queries/0_stateless/02425_categorical_information_value_properties.sql new file mode 100644 index 00000000000..81ed8400680 --- /dev/null +++ b/tests/queries/0_stateless/02425_categorical_information_value_properties.sql @@ -0,0 +1,14 @@ +SELECT round(arrayJoin(categoricalInformationValue(x.1, x.2)), 3) FROM (SELECT arrayJoin([(0, 0), (NULL, 2), (1, 0), (1, 1)]) AS x); +SELECT corr(c1, c2) FROM VALUES((0, 0), (NULL, 2), (1, 0), (1, 1)); +SELECT round(arrayJoin(categoricalInformationValue(c1, c2)), 3) FROM VALUES((0, 0), (NULL, 2), (1, 0), (1, 1)); +SELECT round(arrayJoin(categoricalInformationValue(c1, c2)), 3) FROM VALUES((0, 0), (NULL, 1), (1, 0), (1, 1)); +SELECT categoricalInformationValue(c1, c2) FROM VALUES((0, 0), (NULL, 1)); +SELECT categoricalInformationValue(c1, c2) FROM VALUES((NULL, 1)); -- { serverError 43 } +SELECT categoricalInformationValue(dummy, dummy); +SELECT categoricalInformationValue(dummy, dummy) WHERE 0; +SELECT categoricalInformationValue(c1, c2) FROM VALUES((toNullable(0), 0)); +SELECT groupUniqArray(*) FROM VALUES(toNullable(0)); +SELECT groupUniqArray(*) FROM VALUES(NULL); +SELECT categoricalInformationValue(c1, c2) FROM VALUES((NULL, NULL)); -- { serverError 43 } +SELECT categoricalInformationValue(c1, c2) FROM VALUES((0, 0), (NULL, 0)); +SELECT quantiles(0.5, 0.9)(c1) FROM VALUES(0::Nullable(UInt8)); diff --git a/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.reference b/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.reference new file mode 100644 index 00000000000..3bfced8d8bd --- /dev/null +++ b/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.reference @@ -0,0 +1 @@ +4908278 diff --git a/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.sh b/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.sh new file mode 100755 index 00000000000..dc9f1ec8ed2 --- /dev/null +++ b/tests/queries/0_stateless/02426_low_cardinality_fixed_string_insert_field.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_LOCAL} --structure 'x LowCardinality(FixedString(2454139))' --input-format Values --output-format TSV --query "SELECT * FROM table" <<< '(1)' | wc -c diff --git a/tests/queries/0_stateless/02426_orc_bug.reference b/tests/queries/0_stateless/02426_orc_bug.reference new file mode 100644 index 00000000000..e5ad2b49289 Binary files /dev/null and b/tests/queries/0_stateless/02426_orc_bug.reference differ diff --git a/tests/queries/0_stateless/02426_orc_bug.sql b/tests/queries/0_stateless/02426_orc_bug.sql new file mode 100644 index 00000000000..7016f1ceb70 --- /dev/null +++ b/tests/queries/0_stateless/02426_orc_bug.sql @@ -0,0 +1,3 @@ +-- Tags: no-fasttest + +SELECT arrayJoin([[], [1]]) FORMAT ORC; diff --git a/tests/queries/0_stateless/02426_pod_array_overflow_2.reference b/tests/queries/0_stateless/02426_pod_array_overflow_2.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02426_pod_array_overflow_2.sql b/tests/queries/0_stateless/02426_pod_array_overflow_2.sql new file mode 100644 index 00000000000..52a00730227 --- /dev/null +++ b/tests/queries/0_stateless/02426_pod_array_overflow_2.sql @@ -0,0 +1 @@ +SELECT * FROM format(Native, 'k0\x23Array(Tuple(FixedString(1), Int64))\0\0\0\0\0\0\0�����\0����������������\0�\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0d\0\0\0\0\0\0\0\0\0\0\0\0\0�1?Vi�%'); -- { serverError 128 } diff --git a/tests/queries/0_stateless/02426_pod_array_overflow_3.reference b/tests/queries/0_stateless/02426_pod_array_overflow_3.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02426_pod_array_overflow_3.sql b/tests/queries/0_stateless/02426_pod_array_overflow_3.sql new file mode 100644 index 00000000000..857ba2ca28e --- /dev/null +++ b/tests/queries/0_stateless/02426_pod_array_overflow_3.sql @@ -0,0 +1 @@ +SELECT * FROM format(Native, '\x01\x01\x01x\x0CArray(UInt8)\x01\x00\xBD\xEF\xBF\xBD\xEF\xBF\xBD\xEF'); -- { serverError 128 } diff --git a/tests/queries/0_stateless/02427_column_nullable_ubsan.reference b/tests/queries/0_stateless/02427_column_nullable_ubsan.reference new file mode 100644 index 00000000000..e6c99ff9291 --- /dev/null +++ b/tests/queries/0_stateless/02427_column_nullable_ubsan.reference @@ -0,0 +1,10 @@ +0 999999 999999 +0 999998 999998 +0 999997 999997 +0 999996 999996 +0 999995 999995 +0 999994 999994 +0 999993 999993 +0 999992 999992 +0 999991 999991 +0 999990 999990 diff --git a/tests/queries/0_stateless/02427_column_nullable_ubsan.sql b/tests/queries/0_stateless/02427_column_nullable_ubsan.sql new file mode 100644 index 00000000000..3d1a51804a7 --- /dev/null +++ b/tests/queries/0_stateless/02427_column_nullable_ubsan.sql @@ -0,0 +1 @@ +SELECT * FROM (SELECT * FROM (SELECT 0 AS a, toNullable(number) AS b, toString(number) AS c FROM numbers(1000000.)) ORDER BY a DESC, b DESC, c ASC LIMIT 1500) LIMIT 10; diff --git a/tests/queries/0_stateless/02427_msan_group_array_resample.reference b/tests/queries/0_stateless/02427_msan_group_array_resample.reference new file mode 100644 index 00000000000..1a7f33e1d69 --- /dev/null +++ b/tests/queries/0_stateless/02427_msan_group_array_resample.reference @@ -0,0 +1 @@ +[[10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]] diff --git a/tests/queries/0_stateless/02427_msan_group_array_resample.sql b/tests/queries/0_stateless/02427_msan_group_array_resample.sql new file mode 100644 index 00000000000..6eccf59a6af --- /dev/null +++ b/tests/queries/0_stateless/02427_msan_group_array_resample.sql @@ -0,0 +1 @@ +SELECT arrayMap(x -> finalizeAggregation(x), state) FROM (SELECT groupArrayResample(9223372036854775806, 1048575, 65537)(number, number % 3), groupArrayStateResample(10, 2147483648, 65535)(number, number % 9223372036854775806) AS state FROM numbers(100)); diff --git a/tests/queries/0_stateless/02427_mutate_and_zero_copy_replication_zookeeper.reference b/tests/queries/0_stateless/02427_mutate_and_zero_copy_replication_zookeeper.reference new file mode 100644 index 00000000000..8717cc1b04e --- /dev/null +++ b/tests/queries/0_stateless/02427_mutate_and_zero_copy_replication_zookeeper.reference @@ -0,0 +1 @@ +2 1 1 diff --git a/tests/queries/0_stateless/02427_mutate_and_zero_copy_replication_zookeeper.sql b/tests/queries/0_stateless/02427_mutate_and_zero_copy_replication_zookeeper.sql new file mode 100644 index 00000000000..9b0a52b8dbd --- /dev/null +++ b/tests/queries/0_stateless/02427_mutate_and_zero_copy_replication_zookeeper.sql @@ -0,0 +1,40 @@ +DROP TABLE IF EXISTS mutate_and_zero_copy_replication1; +DROP TABLE IF EXISTS mutate_and_zero_copy_replication2; + +CREATE TABLE mutate_and_zero_copy_replication1 +( + a UInt64, + b String, + c Float64 +) +ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_02427_mutate_and_zero_copy_replication/alter', '1') +ORDER BY tuple() +SETTINGS old_parts_lifetime=0, cleanup_delay_period=300, cleanup_delay_period_random_add=300, min_bytes_for_wide_part = 0; + +CREATE TABLE mutate_and_zero_copy_replication2 +( + a UInt64, + b String, + c Float64 +) +ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_02427_mutate_and_zero_copy_replication/alter', '2') +ORDER BY tuple() +SETTINGS old_parts_lifetime=0, cleanup_delay_period=300, cleanup_delay_period_random_add=300; + + +INSERT INTO mutate_and_zero_copy_replication1 VALUES (1, '1', 1.0); +SYSTEM SYNC REPLICA mutate_and_zero_copy_replication2; + +SET mutations_sync=2; + +ALTER TABLE mutate_and_zero_copy_replication1 UPDATE a = 2 WHERE 1; + +DROP TABLE mutate_and_zero_copy_replication1 SYNC; + +DETACH TABLE mutate_and_zero_copy_replication2; +ATTACH TABLE mutate_and_zero_copy_replication2; + +SELECT * FROM mutate_and_zero_copy_replication2 WHERE NOT ignore(*); + +DROP TABLE IF EXISTS mutate_and_zero_copy_replication1; +DROP TABLE IF EXISTS mutate_and_zero_copy_replication2; diff --git a/tests/queries/0_stateless/02428_batch_nullable_assert.reference b/tests/queries/0_stateless/02428_batch_nullable_assert.reference new file mode 100644 index 00000000000..823ebfc421c --- /dev/null +++ b/tests/queries/0_stateless/02428_batch_nullable_assert.reference @@ -0,0 +1 @@ +100,-9223372036854775808,nan diff --git a/tests/queries/0_stateless/02428_batch_nullable_assert.sql b/tests/queries/0_stateless/02428_batch_nullable_assert.sql new file mode 100644 index 00000000000..eed9b9d34c7 --- /dev/null +++ b/tests/queries/0_stateless/02428_batch_nullable_assert.sql @@ -0,0 +1,24 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/41470 +SELECT + roundBankers(100), + -9223372036854775808, + roundBankers(result.2, 256) +FROM + ( + SELECT studentTTest(sample, variant) AS result + FROM + ( + SELECT + toFloat64(number) % NULL AS sample, + 0 AS variant + FROM system.numbers + LIMIT 1025 + UNION ALL + SELECT + (toFloat64(number) % 9223372036854775807) + nan AS sample, + -9223372036854775808 AS variant + FROM system.numbers + LIMIT 1024 + ) + ) +FORMAT CSV diff --git a/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.reference b/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh b/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh new file mode 100755 index 00000000000..11ca3f43d8f --- /dev/null +++ b/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Tags: long, no-backward-compatibility-check + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Test that running distributed query and cancel it ASAP, +# this can trigger a hung/deadlock in ProcessorList. +for i in {1..100}; do + query_id="$CLICKHOUSE_TEST_UNIQUE_NAME-$i" + $CLICKHOUSE_CLIENT --format Null --query_id "$query_id" -q "select * from remote('127.{1|2|3|4|5|6}', numbers(1e12))" 2>/dev/null & + while :; do + killed_queries="$($CLICKHOUSE_CLIENT -q "kill query where query_id = '$query_id' sync" | wc -l)" + if [[ "$killed_queries" -ge 1 ]]; then + break + fi + done + wait -n + query_return_status=$? + if [[ $query_return_status -eq 0 ]]; then + echo "Query $query_id should be cancelled, however it returns successfully" + fi +done diff --git a/utils/changelog/README.md b/utils/changelog/README.md index 8218af83d96..739229b49c9 100644 --- a/utils/changelog/README.md +++ b/utils/changelog/README.md @@ -13,6 +13,8 @@ python3 changelog.py -h Usage example: ``` +git fetch --tags # changelog.py depends on having the tags available, this will fetch them + python3 changelog.py --output=changelog-v22.4.1.2305-prestable.md --gh-user-or-token="$GITHUB_TOKEN" v21.6.2.7-prestable python3 changelog.py --output=changelog-v22.4.1.2305-prestable.md --gh-user-or-token="$USER" --gh-password="$PASSWORD" v21.6.2.7-prestable ``` diff --git a/utils/list-versions/update-docker-version.sh b/utils/list-versions/update-docker-version.sh new file mode 100755 index 00000000000..429da330a9f --- /dev/null +++ b/utils/list-versions/update-docker-version.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +# We check only our code, that's why we skip contrib +GIT_ROOT=$(git rev-parse --show-cdup) +GIT_ROOT=${GIT_ROOT:-.} +VERSION=$(sed -e '1 s/^v//; 1 s/-.*//p; d' "$GIT_ROOT"/utils/list-versions/version_date.tsv) + +find "$GIT_ROOT/docker/server/" -name 'Dockerfile.*' -print0 | xargs -0 sed -i "/^ARG VERSION=/ s/^.*$/ARG VERSION=\"$VERSION\"/" diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index f2c8cfc4c76..399f6066f15 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v22.8.5.29-lts 2022-09-13 v22.8.4.7-lts 2022-08-31 v22.8.3.13-lts 2022-08-29 v22.8.2.11-lts 2022-08-23 @@ -7,6 +8,7 @@ v22.7.4.16-stable 2022-08-23 v22.7.3.5-stable 2022-08-10 v22.7.2.15-stable 2022-08-03 v22.7.1.2484-stable 2022-07-21 +v22.6.8.35-stable 2022-09-19 v22.6.7.7-stable 2022-08-29 v22.6.6.16-stable 2022-08-23 v22.6.5.22-stable 2022-08-09